X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=blobdiff_plain;f=validators%2Fsrc%2FCaasValidation.py;h=8dc6f6a387d8541c0b7ab1c467cdf760e46082f9;hb=388a4abd6fb9402cb6c8a1efa88e3ba4b4c4718d;hp=a42ef6778d9bcca0c278819adb8f6bf2e3dc8327;hpb=a2f332c01b54a5503333fb36fd6750cef035186d;p=ta%2Fcm-plugins.git diff --git a/validators/src/CaasValidation.py b/validators/src/CaasValidation.py index a42ef67..8dc6f6a 100644 --- a/validators/src/CaasValidation.py +++ b/validators/src/CaasValidation.py @@ -67,24 +67,18 @@ class CaasValidationUtils(object): class CaasValidation(cmvalidator.CMValidator): - - SUBSCRIPTION = r'^cloud\.caas|cloud\.hosts|cloud\.networking$' + SUBSCRIPTION = r'^cloud\.caas|cloud\.hosts|cloud\.networking|cloud\.network_profiles$' CAAS_DOMAIN = 'cloud.caas' - NETW_DOMAIN = 'cloud.networking' HOSTS_DOMAIN = 'cloud.hosts' + NETW_DOMAIN = 'cloud.networking' + NETPROF_DOMAIN = 'cloud.network_profiles' SERV_PROF = 'service_profiles' CAAS_PROFILE_PATTERN = 'caas_master|caas_worker' CIDR = 'cidr' - DOCKER_SIZE_QOUTA = "docker_size_quota" - DOCKER_SIZE_QOUTA_PATTERN = r"^\d*[G,M,K]$" - - CHART_NAME = "chart_name" - CHART_NAME_PATTERN = r"[A-Za-z0-9\.-_]+" - - CHART_VERSION = "chart_version" - CHART_VERSION_PATTERN = r"^\d+\.\d+\.\d+$" + DOCKER_SIZE_QUOTA = "docker_size_quota" + DOCKER_SIZE_QUOTA_PATTERN = r"^\d*[G,M,K]$" HELM_OP_TIMEOUT = "helm_operation_timeout" @@ -92,13 +86,22 @@ class CaasValidation(cmvalidator.CMValidator): INSTANTIATION_TIMEOUT = "instantiation_timeout" - HELM_PARAMETERS = "helm_parameters" - ENCRYPTED_CA = "encrypted_ca" ENCRYPTED_CA_KEY = "encrypted_ca_key" + CLUSTER_NETS = 'cluster_networks' + TENANT_NETS = 'tenant_networks' + + BLOG_FORWARDING = "infra_log_store" + LOG_FORWARDING = "log_forwarding" + URL_PORT_PATTERN = r"^(?:https?|udp|tcp):(?:\/\/)(?:((?:[\w\.-]+|" \ + r"\[(([1-9a-f][0-9a-f]{0,3}|\:)\:[1-9a-f][0-9a-f]{0,3}){0,7}\])\:[0-9]+))" + FLUENTD_PLUGINS = ['elasticsearch', 'remote_syslog'] + INFRA_LOG_FLUENTD_PLUGINS = ['elasticsearch', 'remote_syslog'] + LOG_FW_STREAM = ['stdout', 'stderr', 'both'] + DOMAIN_NAME = "dns_domain" - DOMAIN_NAME_PATTERN = "^[a-z0-9]([a-z0-9-\.]{0,253}[a-z0-9])?$" + DOMAIN_NAME_PATTERN = r"^[a-z0-9]([a-z0-9-\.]{0,253}[a-z0-9])?$" def __init__(self): cmvalidator.CMValidator.__init__(self) @@ -117,18 +120,26 @@ class CaasValidation(cmvalidator.CMValidator): return self.props_pre_check(props) self.validate_docker_size_quota() - self.validate_chart_name() - self.validate_chart_version() self.validate_helm_operation_timeout() self.validate_docker0_cidr(props) self.validate_instantiation_timeout() - self.validate_helm_parameters() self.validate_encrypted_ca(self.ENCRYPTED_CA) self.validate_encrypted_ca(self.ENCRYPTED_CA_KEY) + self.validate_log_forwarding() + self.validate_networks(props) self.validate_dns_domain() + def _get_conf(self, props, domain): + if props.get(domain): + conf_str = props[domain] + else: + conf_str = self.get_plugin_client().get_property(domain) + return json.loads(conf_str) + def is_caas_mandatory(self, props): - hosts_conf = json.loads(props[self.HOSTS_DOMAIN]) + if not isinstance(props, dict): + raise CaasValidationError('The given input: {} is not a dictionary!'.format(props)) + hosts_conf = self._get_conf(props, self.HOSTS_DOMAIN) service_profiles = self.caas_utils.get_every_key_occurrence(hosts_conf, self.SERV_PROF) pattern = re.compile(self.CAAS_PROFILE_PATTERN) for profile in service_profiles: @@ -137,50 +148,26 @@ class CaasValidation(cmvalidator.CMValidator): return False def props_pre_check(self, props): - if not isinstance(props, dict): - raise CaasValidationError('The given input: {} is not a dictionary!'.format(props)) - if self.CAAS_DOMAIN not in props: - raise CaasValidationError( - '{} configuration is missing from {}!'.format(self.CAAS_DOMAIN, props)) - self.caas_conf = json.loads(props[self.CAAS_DOMAIN]) + self.caas_conf = self._get_conf(props, self.CAAS_DOMAIN) self.conf = {self.CAAS_DOMAIN: self.caas_conf} if not self.caas_conf: raise CaasValidationError('{} is an empty dictionary!'.format(self.conf)) def validate_docker_size_quota(self): - if not self.caas_utils.is_optional_param_present(self.DOCKER_SIZE_QOUTA, self.caas_conf): - return - if not re.match(self.DOCKER_SIZE_QOUTA_PATTERN, self.caas_conf[self.DOCKER_SIZE_QOUTA]): - raise CaasValidationError('{} is not a valid {}!'.format( - self.caas_conf[self.DOCKER_SIZE_QOUTA], - self.DOCKER_SIZE_QOUTA)) - - def validate_chart_name(self): - if not self.caas_utils.is_optional_param_present(self.CHART_NAME, self.caas_conf): - return - if not re.match(self.CHART_NAME_PATTERN, self.caas_conf[self.CHART_NAME]): - raise CaasValidationError('{} is not a valid {} !'.format( - self.caas_conf[self.CHART_NAME], - self.CHART_NAME)) - - def validate_chart_version(self): - if not self.caas_utils.is_optional_param_present(self.CHART_VERSION, self.caas_conf): + if not self.caas_utils.is_optional_param_present(self.DOCKER_SIZE_QUOTA, self.caas_conf): return - if not self.caas_conf[self.CHART_NAME]: - logging.warn('{} shall be set only, when {} is set.'.format( - self.CHART_VERSION, self.CHART_NAME)) - if not re.match(self.CHART_VERSION_PATTERN, self.caas_conf[self.CHART_VERSION]): - raise CaasValidationError('{} is not a valid {} !'.format( - self.caas_conf[self.CHART_VERSION], - self.CHART_VERSION)) + if not re.match(self.DOCKER_SIZE_QUOTA_PATTERN, self.caas_conf[self.DOCKER_SIZE_QUOTA]): + raise CaasValidationError( + '{} is not a valid {}!'.format(self.caas_conf[self.DOCKER_SIZE_QUOTA], + self.DOCKER_SIZE_QUOTA)) def validate_helm_operation_timeout(self): if not self.caas_utils.is_optional_param_present(self.HELM_OP_TIMEOUT, self.caas_conf): return if not isinstance(self.caas_conf[self.HELM_OP_TIMEOUT], int): - raise CaasValidationError('{}:{} is not an integer'.format( - self.HELM_OP_TIMEOUT, - self.caas_conf[self.HELM_OP_TIMEOUT])) + raise CaasValidationError( + '{}:{} is not an integer'.format(self.HELM_OP_TIMEOUT, + self.caas_conf[self.HELM_OP_TIMEOUT])) def get_docker0_cidr_netw_obj(self, subnet): try: @@ -190,14 +177,14 @@ class CaasValidation(cmvalidator.CMValidator): self.DOCKER0_CIDR, exc)) def check_docker0_cidr_overlaps_with_netw_subnets(self, docker0_cidr, props): - netw_conf = json.loads(props[self.NETW_DOMAIN]) + netw_conf = self._get_conf(props, self.NETW_DOMAIN) cidrs = self.caas_utils.get_every_key_occurrence(netw_conf, self.CIDR) for cidr in cidrs: if docker0_cidr.overlaps(ipaddr.IPNetwork(cidr)): raise CaasValidationError( 'CIDR configured for {} shall be an unused IP range, ' - 'but it overlaps with {} from {}.'.format( - self.DOCKER0_CIDR, cidr, self.NETW_DOMAIN)) + 'but it overlaps with {} from {}.'.format(self.DOCKER0_CIDR, cidr, + self.NETW_DOMAIN)) def validate_docker0_cidr(self, props): if not self.caas_utils.is_optional_param_present(self.DOCKER0_CIDR, self.caas_conf): @@ -211,15 +198,7 @@ class CaasValidation(cmvalidator.CMValidator): return if not isinstance(self.caas_conf[self.INSTANTIATION_TIMEOUT], int): raise CaasValidationError('{}:{} is not an integer'.format( - self.INSTANTIATION_TIMEOUT, - self.caas_conf[self.INSTANTIATION_TIMEOUT])) - - def validate_helm_parameters(self): - if not self.caas_utils.is_optional_param_present(self.HELM_PARAMETERS, self.caas_conf): - return - if not isinstance(self.caas_conf[self.HELM_PARAMETERS], dict): - raise CaasValidationError('The given input: {} is not a dictionary!'.format( - self.caas_conf[self.HELM_PARAMETERS])) + self.INSTANTIATION_TIMEOUT, self.caas_conf[self.INSTANTIATION_TIMEOUT])) def validate_encrypted_ca(self, enc_ca): self.caas_utils.check_key_in_dict(enc_ca, self.caas_conf) @@ -231,6 +210,150 @@ class CaasValidation(cmvalidator.CMValidator): except TypeError as exc: raise CaasValidationError('Invalid {}: {}'.format(enc_ca, exc)) + def validate_log_forwarding(self): + # pylint: disable=too-many-branches + if self.caas_utils.is_optional_param_present(self.BLOG_FORWARDING, self.caas_conf): + if self.caas_conf[self.BLOG_FORWARDING] not in self.INFRA_LOG_FLUENTD_PLUGINS: + raise CaasValidationError('"{}" property not valid! ' + 'Choose from {}!'.format(self.BLOG_FORWARDING, + self.INFRA_LOG_FLUENTD_PLUGINS)) + if self.caas_utils.is_optional_param_present(self.LOG_FORWARDING, self.caas_conf): + log_fw_list = self.caas_conf[self.LOG_FORWARDING] + if log_fw_list: + url_d = dict() + url_s = set() + for list_item in log_fw_list: + self.caas_utils.check_key_in_dict('namespace', list_item) + if list_item['namespace'] == 'kube-system': + raise CaasValidationError( + 'You can\'t set "kube-system" as namespace in "{}"!'.format( + self.LOG_FORWARDING)) + self.caas_utils.check_key_in_dict('target_url', list_item) + if not list_item['target_url'] or not re.match(self.URL_PORT_PATTERN, + list_item['target_url']): + raise CaasValidationError( + '"target_url" property {} not valid!'.format(list_item['target_url'])) + if not url_d: + url_d[list_item['namespace']] = list_item['target_url'] + if list_item['namespace'] in url_d: + if list_item['target_url'] in url_s: + raise CaasValidationError('There can\'t be multiple rules for the same ' + 'target_url for the same {} ' + 'namespace!'.format(list_item['namespace'])) + else: + url_s.add(list_item['target_url']) + url_d[list_item['namespace']] = url_s + else: + url_d[list_item['namespace']] = list_item['target_url'] + if self.caas_utils.is_optional_param_present('plugin', list_item) and list_item[ + 'plugin'] not in self.FLUENTD_PLUGINS: + raise CaasValidationError( + '"plugin" property not valid! Choose from {}'.format( + self.FLUENTD_PLUGINS)) + if self.caas_utils.is_optional_param_present('stream', list_item) and list_item[ + 'stream'] not in self.LOG_FW_STREAM: + raise CaasValidationError( + '"stream" property not valid! Choose from {}'.format( + self.LOG_FW_STREAM)) + + def validate_networks(self, props): + caas_nets = [] + for nets_key in [self.CLUSTER_NETS, self.TENANT_NETS]: + if self.caas_utils.is_optional_param_present(nets_key, self.caas_conf): + if not isinstance(self.caas_conf[nets_key], list): + raise CaasValidationError('{} is not a list'.format(nets_key)) + if len(set(self.caas_conf[nets_key])) != len(self.caas_conf[nets_key]): + raise CaasValidationError('{} has duplicate entries'.format(nets_key)) + caas_nets.extend(self.caas_conf[nets_key]) + if len(set(caas_nets)) != len(caas_nets): + raise CaasValidationError('{} and {} must be distinct, but same entries are ' + 'found from both lists'.format(self.CLUSTER_NETS, + self.TENANT_NETS)) + self._validate_homogenous_net_setup(props, caas_nets) + + def _validate_homogenous_net_setup(self, props, caas_nets): + # Validate homogenous CaaS provider network setup + # pylint: disable=too-many-locals,too-many-nested-blocks + hosts_conf = self._get_conf(props, self.HOSTS_DOMAIN) + netprof_conf = self._get_conf(props, self.NETPROF_DOMAIN) + net_iface_map = {} + for net in caas_nets: + net_iface_map[net] = None + for host, host_conf in hosts_conf.iteritems(): + # Validate only nodes that can host containerized workloads + if ('caas_worker' in host_conf[self.SERV_PROF] or + ('caas_master' in host_conf[self.SERV_PROF] and + 'compute' not in host_conf[self.SERV_PROF])): + # Validating CaaS network 'net' mapping in 'host' + profiles = host_conf.get('network_profiles') + if isinstance(profiles, list) and profiles: + net_prof = netprof_conf.get(profiles[0]) + if net_prof is not None: + ifaces = net_prof.get('provider_network_interfaces', {}) + caas_provider_interfaces = self._filter_provider_networks_by_type( + self._filter_provider_networkinterfaces_by_net(ifaces, net), 'caas') + sriov_networks = net_prof.get('sriov_provider_networks', {}) + caas_sriov_networks_present = bool( + net in sriov_networks and + sriov_networks[net].get('type', "") == 'caas') + if not caas_provider_interfaces and not caas_sriov_networks_present: + raise CaasValidationError('CaaS network {} missing from host {}' + .format(net, host)) + if caas_provider_interfaces: + self._validate_homogenous_provider_net_setup( + net_iface_map, net, ifaces) + if caas_sriov_networks_present: + self._validate_homogenous_sriov_provider_net_setup( + net_iface_map, net, sriov_networks) + + @staticmethod + def _filter_provider_networks_by_type(profile, net_type): + return {name: network for name, network in profile.iteritems() + if network.get('type', "") == net_type} + + @staticmethod + def _filter_provider_networkinterfaces_by_net(provider_interfaces, provider_net): + return {iface: data for iface, data in provider_interfaces.iteritems() + if provider_net in data.get('provider_networks', [])} + + @staticmethod + def _validate_homogenous_provider_net_setup(net_iface_map, net, ifaces): + is_caas_network_present = False + for iface, data in ifaces.iteritems(): + net_type = data.get('type') + networks = data.get('provider_networks', []) + if net in networks and net_type == 'caas': + is_caas_network_present = True + if net_iface_map[net] is None: + net_iface_map[net] = iface + elif net_iface_map[net] != iface: + msg = 'CaaS network {} mapped to interface {} in one host ' + msg += 'and interface {} in another host' + raise CaasValidationError(msg.format(net, iface, + net_iface_map[net])) + break + return is_caas_network_present + + @staticmethod + def _validate_homogenous_sriov_provider_net_setup(net_iface_map, net, sriov_networks): + is_caas_network_present = False + sriov_provider_net = sriov_networks.get(net, {}) + if sriov_provider_net and sriov_provider_net.get('type') == 'caas': + interfaces = sriov_provider_net.get('interfaces', []) + tenant_interfaces = net_iface_map.get(net) + already_used_ifaces = [set(x).intersection(interfaces) + for x in net_iface_map.itervalues() + if x and isinstance(x, list)] + if tenant_interfaces is None and interfaces: + net_iface_map[net] = interfaces + is_caas_network_present = True + elif already_used_ifaces: + msg = 'CaaS network {} mapped to sriov interfaces {} in one host ' + msg += 'and sriov interfaces {} in another host' + raise CaasValidationError(msg.format(net, interfaces, + tenant_interfaces)) + return is_caas_network_present + def validate_dns_domain(self): domain = self.caas_conf[self.DOMAIN_NAME] if not self.caas_utils.is_optional_param_present(self.DOMAIN_NAME, self.caas_conf):