validators/src/CaasValidation.py

   1 #!/usr/bin/python
   2 # Copyright 2019 Nokia
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import json
  17 import re
  18 import base64
  19 import logging
  20 import ipaddr
  21 from cmframework.apis import cmvalidator
  22 from cmdatahandlers.api import validation
  23 from cmdatahandlers.api import configerror
  24
  25
  26 class CaasValidationError(configerror.ConfigError):
  27     def __init__(self, description):
  28         configerror.ConfigError.__init__(
  29             self, 'Validation error in caas_validation: {}'.format(description))
  30
  31
  32 class CaasValidationUtils(object):
  33
  34     def __init__(self):
  35         pass
  36
  37     @staticmethod
  38     def check_key_in_dict(key, dictionary):
  39         if key not in dictionary:
  40             raise CaasValidationError("{} cannot be found in {} ".format(key, dictionary))
  41
  42     def get_every_key_occurrence(self, var, key):
  43         if hasattr(var, 'iteritems'):
  44             for k, v in var.iteritems():
  45                 if k == key:
  46                     yield v
  47                 if isinstance(v, dict):
  48                     for result in self.get_every_key_occurrence(v, key):
  49                         yield result
  50                 elif isinstance(v, list):
  51                     for d in v:
  52                         for result in self.get_every_key_occurrence(d, key):
  53                             yield result
  54
  55     @staticmethod
  56     def is_optional_param_present(key, dictionary):
  57         if key not in dictionary:
  58             logging.info('{} key is not in the config dictionary, since this is an optional '
  59                          'parameter, validation is skipped.'.format(key))
  60             return False
  61         if not dictionary[key]:
  62             logging.info('Although {} key is in the config dictionary the correspondig value is '
  63                          'empty, since this is an optional parametery, '
  64                          'validation is skipped.'.format(key))
  65             return False
  66         return True
  67
  68
  69 class CaasValidation(cmvalidator.CMValidator):
  70     SUBSCRIPTION = r'^cloud\.caas|cloud\.hosts|cloud\.networking|cloud\.network_profiles$'
  71     CAAS_DOMAIN = 'cloud.caas'
  72     HOSTS_DOMAIN = 'cloud.hosts'
  73     NETW_DOMAIN = 'cloud.networking'
  74     NETPROF_DOMAIN = 'cloud.network_profiles'
  75
  76     SERV_PROF = 'service_profiles'
  77     CAAS_PROFILE_PATTERN = 'caas_master|caas_worker'
  78     CIDR = 'cidr'
  79
  80     DOCKER_SIZE_QUOTA = "docker_size_quota"
  81     DOCKER_SIZE_QUOTA_PATTERN = r"^\d*[G,M,K]$"
  82
  83     HELM_OP_TIMEOUT = "helm_operation_timeout"
  84
  85     DOCKER0_CIDR = "docker0_cidr"
  86
  87     INSTANTIATION_TIMEOUT = "instantiation_timeout"
  88
  89     ENCRYPTED_CA = "encrypted_ca"
  90     ENCRYPTED_CA_KEY = "encrypted_ca_key"
  91
  92     CLUSTER_NETS = 'cluster_networks'
  93     TENANT_NETS = 'tenant_networks'
  94
  95     BLOG_FORWARDING = "infra_log_store"
  96     LOG_FORWARDING = "log_forwarding"
  97     URL_PORT_PATTERN = r"^(?:https?|udp|tcp):(?:\/\/)(?:((?:[\w\.-]+|" \
  98                        r"\[(([1-9a-f][0-9a-f]{0,3}|\:)\:[1-9a-f][0-9a-f]{0,3}){0,7}\])\:[0-9]+))"
  99     FLUENTD_PLUGINS = ['elasticsearch', 'remote_syslog']
 100     INFRA_LOG_FLUENTD_PLUGINS = ['elasticsearch', 'remote_syslog']
 101     LOG_FW_STREAM = ['stdout', 'stderr', 'both']
 102
 103     def __init__(self):
 104         cmvalidator.CMValidator.__init__(self)
 105         self.validation_utils = validation.ValidationUtils()
 106         self.conf = None
 107         self.caas_conf = None
 108         self.caas_utils = CaasValidationUtils()
 109
 110     def get_subscription_info(self):
 111         return self.SUBSCRIPTION
 112
 113     def validate_set(self, props):
 114         if not self.is_caas_mandatory(props):
 115             logging.info("{} not found in {}, caas validation is not needed.".format(
 116                 self.CAAS_PROFILE_PATTERN, self.HOSTS_DOMAIN))
 117             return
 118         self.props_pre_check(props)
 119         self.validate_docker_size_quota()
 120         self.validate_helm_operation_timeout()
 121         self.validate_docker0_cidr(props)
 122         self.validate_instantiation_timeout()
 123         self.validate_encrypted_ca(self.ENCRYPTED_CA)
 124         self.validate_encrypted_ca(self.ENCRYPTED_CA_KEY)
 125         self.validate_log_forwarding()
 126         self.validate_networks(props)
 127
 128     def _get_conf(self, props, domain):
 129         if props.get(domain):
 130             conf_str = props[domain]
 131         else:
 132             conf_str = self.get_plugin_client().get_property(domain)
 133         return json.loads(conf_str)
 134
 135     def is_caas_mandatory(self, props):
 136         if not isinstance(props, dict):
 137             raise CaasValidationError('The given input: {} is not a dictionary!'.format(props))
 138         hosts_conf = self._get_conf(props, self.HOSTS_DOMAIN)
 139         service_profiles = self.caas_utils.get_every_key_occurrence(hosts_conf, self.SERV_PROF)
 140         pattern = re.compile(self.CAAS_PROFILE_PATTERN)
 141         for profile in service_profiles:
 142             if filter(pattern.match, profile):
 143                 return True
 144         return False
 145
 146     def props_pre_check(self, props):
 147         self.caas_conf = self._get_conf(props, self.CAAS_DOMAIN)
 148         self.conf = {self.CAAS_DOMAIN: self.caas_conf}
 149         if not self.caas_conf:
 150             raise CaasValidationError('{} is an empty dictionary!'.format(self.conf))
 151
 152     def validate_docker_size_quota(self):
 153         if not self.caas_utils.is_optional_param_present(self.DOCKER_SIZE_QUOTA, self.caas_conf):
 154             return
 155         if not re.match(self.DOCKER_SIZE_QUOTA_PATTERN, self.caas_conf[self.DOCKER_SIZE_QUOTA]):
 156             raise CaasValidationError(
 157                 '{} is not a valid {}!'.format(self.caas_conf[self.DOCKER_SIZE_QUOTA],
 158                                                self.DOCKER_SIZE_QUOTA))
 159
 160     def validate_helm_operation_timeout(self):
 161         if not self.caas_utils.is_optional_param_present(self.HELM_OP_TIMEOUT, self.caas_conf):
 162             return
 163         if not isinstance(self.caas_conf[self.HELM_OP_TIMEOUT], int):
 164             raise CaasValidationError(
 165                 '{}:{} is not an integer'.format(self.HELM_OP_TIMEOUT,
 166                                                  self.caas_conf[self.HELM_OP_TIMEOUT]))
 167
 168     def get_docker0_cidr_netw_obj(self, subnet):
 169         try:
 170             return ipaddr.IPNetwork(subnet)
 171         except ValueError as exc:
 172             raise CaasValidationError('{} is an invalid subnet address: {}'.format(
 173                 self.DOCKER0_CIDR, exc))
 174
 175     def check_docker0_cidr_overlaps_with_netw_subnets(self, docker0_cidr, props):
 176         netw_conf = self._get_conf(props, self.NETW_DOMAIN)
 177         cidrs = self.caas_utils.get_every_key_occurrence(netw_conf, self.CIDR)
 178         for cidr in cidrs:
 179             if docker0_cidr.overlaps(ipaddr.IPNetwork(cidr)):
 180                 raise CaasValidationError(
 181                     'CIDR configured for {} shall be an unused IP range, '
 182                     'but it overlaps with {} from {}.'.format(self.DOCKER0_CIDR, cidr,
 183                                                               self.NETW_DOMAIN))
 184
 185     def validate_docker0_cidr(self, props):
 186         if not self.caas_utils.is_optional_param_present(self.DOCKER0_CIDR, self.caas_conf):
 187             return
 188         docker0_cidr_obj = self.get_docker0_cidr_netw_obj(self.caas_conf[self.DOCKER0_CIDR])
 189         self.check_docker0_cidr_overlaps_with_netw_subnets(docker0_cidr_obj, props)
 190
 191     def validate_instantiation_timeout(self):
 192         if not self.caas_utils.is_optional_param_present(self.INSTANTIATION_TIMEOUT,
 193                                                          self.caas_conf):
 194             return
 195         if not isinstance(self.caas_conf[self.INSTANTIATION_TIMEOUT], int):
 196             raise CaasValidationError('{}:{} is not an integer'.format(
 197                 self.INSTANTIATION_TIMEOUT, self.caas_conf[self.INSTANTIATION_TIMEOUT]))
 198
 199     def validate_encrypted_ca(self, enc_ca):
 200         self.caas_utils.check_key_in_dict(enc_ca, self.caas_conf)
 201         enc_ca_str = self.caas_conf[enc_ca][0]
 202         if not enc_ca_str:
 203             raise CaasValidationError('{} shall not be empty !'.format(enc_ca))
 204         try:
 205             base64.b64decode(enc_ca_str)
 206         except TypeError as exc:
 207             raise CaasValidationError('Invalid {}: {}'.format(enc_ca, exc))
 208
 209     def validate_log_forwarding(self):
 210         # pylint: disable=too-many-branches
 211         if self.caas_utils.is_optional_param_present(self.BLOG_FORWARDING, self.caas_conf):
 212             if self.caas_conf[self.BLOG_FORWARDING] not in self.INFRA_LOG_FLUENTD_PLUGINS:
 213                 raise CaasValidationError('"{}" property not valid! '
 214                                           'Choose from {}!'.format(self.BLOG_FORWARDING,
 215                                                                    self.INFRA_LOG_FLUENTD_PLUGINS))
 216         if self.caas_utils.is_optional_param_present(self.LOG_FORWARDING, self.caas_conf):
 217             log_fw_list = self.caas_conf[self.LOG_FORWARDING]
 218             if log_fw_list:
 219                 url_d = dict()
 220                 url_s = set()
 221                 for list_item in log_fw_list:
 222                     self.caas_utils.check_key_in_dict('namespace', list_item)
 223                     if list_item['namespace'] == 'kube-system':
 224                         raise CaasValidationError(
 225                             'You can\'t set "kube-system" as namespace in "{}"!'.format(
 226                                 self.LOG_FORWARDING))
 227                     self.caas_utils.check_key_in_dict('target_url', list_item)
 228                     if not list_item['target_url'] or not re.match(self.URL_PORT_PATTERN,
 229                                                                    list_item['target_url']):
 230                         raise CaasValidationError(
 231                             '"target_url" property {} not valid!'.format(list_item['target_url']))
 232                     if not url_d:
 233                         url_d[list_item['namespace']] = list_item['target_url']
 234                     if list_item['namespace'] in url_d:
 235                         if list_item['target_url'] in url_s:
 236                             raise CaasValidationError('There can\'t be multiple rules for the same '
 237                                                       'target_url for the same {} '
 238                                                       'namespace!'.format(list_item['namespace']))
 239                         else:
 240                             url_s.add(list_item['target_url'])
 241                         url_d[list_item['namespace']] = url_s
 242                     else:
 243                         url_d[list_item['namespace']] = list_item['target_url']
 244                     if self.caas_utils.is_optional_param_present('plugin', list_item) and list_item[
 245                         'plugin'] not in self.FLUENTD_PLUGINS:
 246                         raise CaasValidationError(
 247                             '"plugin" property not valid! Choose from {}'.format(
 248                                 self.FLUENTD_PLUGINS))
 249                     if self.caas_utils.is_optional_param_present('stream', list_item) and list_item[
 250                         'stream'] not in self.LOG_FW_STREAM:
 251                         raise CaasValidationError(
 252                             '"stream" property not valid! Choose from {}'.format(
 253                                 self.LOG_FW_STREAM))
 254
 255     def validate_networks(self, props):
 256         caas_nets = []
 257         for nets_key in [self.CLUSTER_NETS, self.TENANT_NETS]:
 258             if self.caas_utils.is_optional_param_present(nets_key, self.caas_conf):
 259                 if not isinstance(self.caas_conf[nets_key], list):
 260                     raise CaasValidationError('{} is not a list'.format(nets_key))
 261                 if len(set(self.caas_conf[nets_key])) != len(self.caas_conf[nets_key]):
 262                     raise CaasValidationError('{} has duplicate entries'.format(nets_key))
 263                 caas_nets.extend(self.caas_conf[nets_key])
 264         if len(set(caas_nets)) != len(caas_nets):
 265             raise CaasValidationError('{} and {} must be distinct, but same entries are '
 266                                       'found from both lists'.format(self.CLUSTER_NETS,
 267                                                                      self.TENANT_NETS))
 268         self._validate_homogenous_net_setup(props, caas_nets)
 269
 270     def _validate_homogenous_net_setup(self, props, caas_nets):
 271         # Validate homogenous CaaS provider network setup
 272         # pylint: disable=too-many-locals,too-many-nested-blocks
 273         hosts_conf = self._get_conf(props, self.HOSTS_DOMAIN)
 274         netprof_conf = self._get_conf(props, self.NETPROF_DOMAIN)
 275         net_iface_map = {}
 276         for net in caas_nets:
 277             net_iface_map[net] = None
 278             for host, host_conf in hosts_conf.iteritems():
 279                 # Validate only nodes that can host containerized workloads
 280                 if ('caas_worker' in host_conf[self.SERV_PROF] or
 281                         ('caas_master' in host_conf[self.SERV_PROF] and
 282                          'compute' not in host_conf[self.SERV_PROF])):
 283                     # Validating CaaS network 'net' mapping in 'host'
 284                     is_caas_network_present = False
 285                     profiles = host_conf.get('network_profiles')
 286                     if isinstance(profiles, list) and profiles:
 287                         net_prof = netprof_conf.get(profiles[0])
 288                     if net_prof is not None:
 289                         ifaces = net_prof.get('provider_network_interfaces', {})
 290                         for iface, data in ifaces.iteritems():
 291                             net_type = data.get('type')
 292                             networks = data.get('provider_networks', [])
 293                             if net in networks and net_type == 'caas':
 294                                 is_caas_network_present = True
 295                                 if net_iface_map[net] is None:
 296                                     net_iface_map[net] = iface
 297                                 elif net_iface_map[net] != iface:
 298                                     msg = 'CaaS network {} mapped to interface {} in one host '
 299                                     msg += 'and interface {} in another host'
 300                                     raise CaasValidationError(msg.format(net, iface,
 301                                                                          net_iface_map[net]))
 302                                 break
 303                     if not is_caas_network_present:
 304                         raise CaasValidationError('CaaS network {} missing from host {}'
 305                                                   .format(net, host))