cluster/cluster.py

   1 # Copyright 2019 Nokia
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import logging
  16 import six
  17 from crl.remotesession.remotesession import RemoteSession
  18 from crl.interactivesessions.shells.bashshell import BashShell
  19 from hostcli import HostCli
  20 from .envcreator import EnvCreator
  21 from .usermanager import UserManager
  22 from .metasingleton import MetaSingleton
  23 from .hosts import (
  24     MgmtTarget,
  25     Master,
  26     NonMaster,
  27     Profiles,
  28     HostConfig)
  29
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class ClusterError(Exception):
  35     pass
  36
  37
  38 @six.add_metaclass(MetaSingleton)
  39 class Cluster(object):
  40     """Singleton container for NCIR cluster hosts, their service profiles and
  41     access details.
  42     """
  43     def __init__(self):
  44         self._mgmt_target = None
  45         self._hosts = None
  46         self._configprops_cache = None
  47         self._usermanager = UserManager(hostcli_factory=self.create_hostcli)
  48         self._envcreator = EnvCreator(remotesession_factory=self.create_remotesession,
  49                                       usermanager=self._usermanager)
  50
  51     def clear_cache(self):
  52         """Clears configprops cache."""
  53         self._configprops_cache = None
  54
  55     def get_hosts(self):
  56         """Return Host containers of cluster.
  57         """
  58         return [h for _, h in self._hosts.items()]
  59
  60     def initialize(self, host, user, password):
  61         """Initialize Cluster with management VIP
  62
  63         Arguments:
  64             host:  Management VIP IP address
  65             user: Login username
  66             password: password for the user
  67         """
  68         self._mgmt_target = MgmtTarget(host=host, user=user, password=password)
  69         self._hosts = {hc.name: self._create_host(hc) for hc in self._host_configs()}
  70
  71     @staticmethod
  72     def set_profiles(master, worker):
  73         """Set *service_profile' names of *master* and *worker* nodes
  74         """
  75         Profiles.set_profiles(master=master, worker=worker)
  76
  77     def get_mgmt_shelldicts(self):
  78         """Return management VIP shelldicts for RemoteRunner.
  79         """
  80         return [self._mgmt_target.asdict()]
  81
  82     def get_host(self, hostname):
  83         """Get Host container for hostname."""
  84         return self._hosts[hostname]
  85
  86     def get_hosts_with_profiles(self, *service_profiles):
  87         """Get host names matching exactly *service_profiles*.
  88         """
  89         return sorted(list(self._get_hosts_with_profs_gen(set(service_profiles))))
  90
  91     def get_hosts_containing(self, *service_profiles):
  92         """Get host names containing all  *service_profiles*.
  93         """
  94         return sorted(list(self._get_hosts_containing_gen(set(service_profiles))))
  95
  96     def create_remotesession(self):
  97         """Create initialized *RemoteSession* instance.
  98         """
  99         r = RemoteSession()
 100         self.initialize_remotesession(r)
 101         return r
 102
 103     def initialize_remotesession(self, remotesession):
 104         """Initialize :class:`crl.remotesession.remotesession.RemoteSession` instance
 105         with *shelldicts* and *name* of the hosts and sudo-<name> via *set_runner_target*.
 106         Initialize *default* target with *get_mgmt_shelldicts* return value.
 107         The sudo-<name> targets are terminals in target after executed roughly *sudo bash*.
 108         """
 109         self._set_mgmt_targets(remotesession)
 110         for host in self.get_hosts():
 111             remotesession.set_runner_target(shelldicts=host.shelldicts,
 112                                             name=host.name)
 113             remotesession.set_runner_target(
 114                 shelldicts=self._get_sudoshelldicts(host.shelldicts),
 115                 name='sudo-{}'.format(host.name))
 116
 117         remotesession.set_envcreator(self._envcreator)
 118
 119     def _set_mgmt_targets(self, remotesession):
 120         remotesession.set_runner_target(self.get_mgmt_shelldicts())
 121         remotesession.set_runner_target(
 122             shelldicts=self._get_sudoshelldicts(self.get_mgmt_shelldicts()),
 123             name='sudo-default')
 124         remotesession.set_target(host=self._mgmt_target.host,
 125                                  username=self._mgmt_target.user,
 126                                  password=self._mgmt_target.password,
 127                                  name='remotescript-default')
 128
 129     @staticmethod
 130     def _get_sudoshelldicts(shelldicts):
 131         return shelldicts + [{'shellname': BashShell.__name__,
 132                               'cmd': 'sudo bash'}]
 133
 134     def create_hostcli(self):
 135         """Create initialized *HostCli* instance.
 136         """
 137         n = HostCli()
 138         self.initialize_hostcli(n)
 139         return n
 140
 141     def initialize_hostcli(self, hostcli):
 142         """Initialize :class:`crl.hostcli.HostCli` instance
 143         (or :class:`crl.hostcli.OpenStack`) with
 144         :class:`crl.remotesession.remotesession.RemoteSession` instance
 145         initialized with :meth:`.initialize_remotesession`.
 146         """
 147         hostcli.initialize(self.create_remotesession())
 148
 149     def create_user_with_roles(self, *roles):
 150         """Create according to roles list.
 151
 152         Special roles:
 153
 154             all_roles: all roles in the system
 155             no_roles: empty role list
 156
 157         Return:
 158             UserRecord of created user
 159         """
 160         return self._usermanager.create_user_with_roles(*roles)
 161
 162     def delete_users(self):
 163         """Delete all users created by *Cluster*.
 164         """
 165         self._usermanager.delete_users()
 166
 167     def is_dpdk(self):
 168         """Return *True* if *dpdk* is used in provider network interfaces.
 169         More detail, *True* if and only if there is at least one host with
 170         network profile providing *ovs-dpdk* type interface.
 171         """
 172         for _, h in self._hosts.items():
 173             if h.is_dpdk:
 174                 return True
 175         return False
 176
 177     def get_hosts_with_dpdk(self):
 178         """Returns list of hosts where *dpdk* is in use.
 179            In more detail, return sorted list of host names in which there is
 180            at least one network profile containing *dpdk* type interface.
 181         """
 182         return sorted([h.name for _, h in self._hosts.items() if h.is_dpdk])
 183
 184     def _get_hosts_with_profs_gen(self, service_profiles):
 185         def filt(host):
 186             mask = Profiles().profiles_mask
 187             return not (service_profiles ^ set(host.service_profiles)) & mask
 188
 189         return self._filtered_hostnames(filt)
 190
 191     def _get_hosts_containing_gen(self, service_profiles):
 192         def filt(host):
 193             return service_profiles.issubset(set(host.service_profiles))
 194
 195         return self._filtered_hostnames(filt)
 196
 197     def _filtered_hostnames(self, filt):
 198         for h in self.get_hosts():
 199             if filt(h):
 200                 yield h.name
 201
 202     @staticmethod
 203     def _create_host(host_config):
 204         LOGGER.debug('host_config: %s', host_config)
 205         master = Profiles().master
 206         return (Master(host_config)
 207                 if master in host_config.service_profiles else
 208                 NonMaster(host_config))
 209
 210     def _host_configs(self):
 211         LOGGER.debug('cloud_hosts: %s', self._cloud_hosts)
 212         for hostname, v in self._cloud_hosts.items():
 213             yield HostConfig(name=hostname,
 214                              network_domain=v['network_domain'],
 215                              service_profiles=v['service_profiles'],
 216                              networking=self._get_networking(hostname),
 217                              mgmt_target=self._mgmt_target,
 218                              is_dpdk=self._is_host_dpdk(v))
 219
 220     def _is_host_dpdk(self, host_prop):
 221         network_profiles = set(host_prop['network_profiles'])
 222         return bool(network_profiles.intersection(set(self._dpdk_profiles())))
 223
 224     def _dpdk_profiles(self):
 225         profiles = self._get_value_for_prop('cloud.network_profiles')
 226         for prof_n, prof_v in profiles.items():
 227             ifaces = prof_v.get('provider_network_interfaces', {})
 228             for _, iface_v in ifaces.items():
 229                 if iface_v['type'] == 'ovs-dpdk':
 230                     yield prof_n
 231
 232     @property
 233     def _cloud_hosts(self):
 234         return self._get_value_for_prop('cloud.hosts')
 235
 236     def _get_networking(self, hostname):
 237         return self._get_value_for_prop('{}.networking'.format(hostname))
 238
 239     def _get_value_for_prop(self, prop):
 240         for pv in self._configprops:
 241             if pv['property'] == prop:
 242                 return pv['value']
 243         raise ClusterError('Property not found: {}'.format(prop))
 244
 245     @property
 246     def _configprops(self):
 247         if self._configprops_cache is None:
 248             self._setup_configprops_cache()
 249         return self._configprops_cache
 250
 251     def _setup_configprops_cache(self):
 252         s = RemoteSession()
 253         s.set_runner_target(self.get_mgmt_shelldicts())
 254         n = HostCli()
 255         n.initialize(s)
 256         self._configprops_cache = n.run('config-manager list properties')