robot tcs, test charts, robot container added
[ta/cloudtaf.git] / cluster / cluster.py
1 # Copyright 2019 Nokia
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import logging
16 import six
17 from crl.remotesession.remotesession import RemoteSession
18 from crl.interactivesessions.shells.bashshell import BashShell
19 from hostcli import HostCli
20 from .envcreator import EnvCreator
21 from .usermanager import UserManager
22 from .metasingleton import MetaSingleton
23 from .hosts import (
24     MgmtTarget,
25     Master,
26     NonMaster,
27     Profiles,
28     HostConfig)
29
30
31 LOGGER = logging.getLogger(__name__)
32
33
34 class ClusterError(Exception):
35     pass
36
37
38 @six.add_metaclass(MetaSingleton)
39 class Cluster(object):
40     """Singleton container for NCIR cluster hosts, their service profiles and
41     access details.
42     """
43     def __init__(self):
44         self._mgmt_target = None
45         self._hosts = None
46         self._configprops_cache = None
47         self._usermanager = UserManager(hostcli_factory=self.create_hostcli)
48         self._envcreator = EnvCreator(remotesession_factory=self.create_remotesession,
49                                       usermanager=self._usermanager)
50
51     def clear_cache(self):
52         """Clears configprops cache."""
53         self._configprops_cache = None
54
55     def get_hosts(self):
56         """Return Host containers of cluster.
57         """
58         return [h for _, h in self._hosts.items()]
59
60     def initialize(self, host, user, password):
61         """Initialize Cluster with management VIP
62
63         Arguments:
64             host:  Management VIP IP address
65             user: Login username
66             password: password for the user
67         """
68         self._mgmt_target = MgmtTarget(host=host, user=user, password=password)
69         self._hosts = {hc.name: self._create_host(hc) for hc in self._host_configs()}
70
71     @staticmethod
72     def set_profiles(master, worker):
73         """Set *service_profile' names of *master* and *worker* nodes
74         """
75         Profiles.set_profiles(master=master, worker=worker)
76
77     def get_mgmt_shelldicts(self):
78         """Return management VIP shelldicts for RemoteRunner.
79         """
80         return [self._mgmt_target.asdict()]
81
82     def get_host(self, hostname):
83         """Get Host container for hostname."""
84         return self._hosts[hostname]
85
86     def get_hosts_with_profiles(self, *service_profiles):
87         """Get host names matching exactly *service_profiles*.
88         """
89         return sorted(list(self._get_hosts_with_profs_gen(set(service_profiles))))
90
91     def get_hosts_containing(self, *service_profiles):
92         """Get host names containing all  *service_profiles*.
93         """
94         return sorted(list(self._get_hosts_containing_gen(set(service_profiles))))
95
96     def create_remotesession(self):
97         """Create initialized *RemoteSession* instance.
98         """
99         r = RemoteSession()
100         self.initialize_remotesession(r)
101         return r
102
103     def initialize_remotesession(self, remotesession):
104         """Initialize :class:`crl.remotesession.remotesession.RemoteSession` instance
105         with *shelldicts* and *name* of the hosts and sudo-<name> via *set_runner_target*.
106         Initialize *default* target with *get_mgmt_shelldicts* return value.
107         The sudo-<name> targets are terminals in target after executed roughly *sudo bash*.
108         """
109         self._set_mgmt_targets(remotesession)
110         for host in self.get_hosts():
111             remotesession.set_runner_target(shelldicts=host.shelldicts,
112                                             name=host.name)
113             remotesession.set_runner_target(
114                 shelldicts=self._get_sudoshelldicts(host.shelldicts),
115                 name='sudo-{}'.format(host.name))
116
117         remotesession.set_envcreator(self._envcreator)
118
119     def _set_mgmt_targets(self, remotesession):
120         remotesession.set_runner_target(self.get_mgmt_shelldicts())
121         remotesession.set_runner_target(
122             shelldicts=self._get_sudoshelldicts(self.get_mgmt_shelldicts()),
123             name='sudo-default')
124         remotesession.set_target(host=self._mgmt_target.host,
125                                  username=self._mgmt_target.user,
126                                  password=self._mgmt_target.password,
127                                  name='remotescript-default')
128
129     @staticmethod
130     def _get_sudoshelldicts(shelldicts):
131         return shelldicts + [{'shellname': BashShell.__name__,
132                               'cmd': 'sudo bash'}]
133
134     def create_hostcli(self):
135         """Create initialized *HostCli* instance.
136         """
137         n = HostCli()
138         self.initialize_hostcli(n)
139         return n
140
141     def initialize_hostcli(self, hostcli):
142         """Initialize :class:`crl.hostcli.HostCli` instance
143         (or :class:`crl.hostcli.OpenStack`) with
144         :class:`crl.remotesession.remotesession.RemoteSession` instance
145         initialized with :meth:`.initialize_remotesession`.
146         """
147         hostcli.initialize(self.create_remotesession())
148
149     def create_user_with_roles(self, *roles):
150         """Create according to roles list.
151
152         Special roles:
153
154             all_roles: all roles in the system
155             no_roles: empty role list
156
157         Return:
158             UserRecord of created user
159         """
160         return self._usermanager.create_user_with_roles(*roles)
161
162     def delete_users(self):
163         """Delete all users created by *Cluster*.
164         """
165         self._usermanager.delete_users()
166
167     def is_dpdk(self):
168         """Return *True* if *dpdk* is used in provider network interfaces.
169         More detail, *True* if and only if there is at least one host with
170         network profile providing *ovs-dpdk* type interface.
171         """
172         for _, h in self._hosts.items():
173             if h.is_dpdk:
174                 return True
175         return False
176
177     def get_hosts_with_dpdk(self):
178         """Returns list of hosts where *dpdk* is in use.
179            In more detail, return sorted list of host names in which there is
180            at least one network profile containing *dpdk* type interface.
181         """
182         return sorted([h.name for _, h in self._hosts.items() if h.is_dpdk])
183
184     def _get_hosts_with_profs_gen(self, service_profiles):
185         def filt(host):
186             mask = Profiles().profiles_mask
187             return not (service_profiles ^ set(host.service_profiles)) & mask
188
189         return self._filtered_hostnames(filt)
190
191     def _get_hosts_containing_gen(self, service_profiles):
192         def filt(host):
193             return service_profiles.issubset(set(host.service_profiles))
194
195         return self._filtered_hostnames(filt)
196
197     def _filtered_hostnames(self, filt):
198         for h in self.get_hosts():
199             if filt(h):
200                 yield h.name
201
202     @staticmethod
203     def _create_host(host_config):
204         LOGGER.debug('host_config: %s', host_config)
205         master = Profiles().master
206         return (Master(host_config)
207                 if master in host_config.service_profiles else
208                 NonMaster(host_config))
209
210     def _host_configs(self):
211         LOGGER.debug('cloud_hosts: %s', self._cloud_hosts)
212         for hostname, v in self._cloud_hosts.items():
213             yield HostConfig(name=hostname,
214                              network_domain=v['network_domain'],
215                              service_profiles=v['service_profiles'],
216                              networking=self._get_networking(hostname),
217                              mgmt_target=self._mgmt_target,
218                              is_dpdk=self._is_host_dpdk(v))
219
220     def _is_host_dpdk(self, host_prop):
221         network_profiles = set(host_prop['network_profiles'])
222         return bool(network_profiles.intersection(set(self._dpdk_profiles())))
223
224     def _dpdk_profiles(self):
225         profiles = self._get_value_for_prop('cloud.network_profiles')
226         for prof_n, prof_v in profiles.items():
227             ifaces = prof_v.get('provider_network_interfaces', {})
228             for _, iface_v in ifaces.items():
229                 if iface_v['type'] == 'ovs-dpdk':
230                     yield prof_n
231
232     @property
233     def _cloud_hosts(self):
234         return self._get_value_for_prop('cloud.hosts')
235
236     def _get_networking(self, hostname):
237         return self._get_value_for_prop('{}.networking'.format(hostname))
238
239     def _get_value_for_prop(self, prop):
240         for pv in self._configprops:
241             if pv['property'] == prop:
242                 return pv['value']
243         raise ClusterError('Property not found: {}'.format(prop))
244
245     @property
246     def _configprops(self):
247         if self._configprops_cache is None:
248             self._setup_configprops_cache()
249         return self._configprops_cache
250
251     def _setup_configprops_cache(self):
252         s = RemoteSession()
253         s.set_runner_target(self.get_mgmt_shelldicts())
254         n = HostCli()
255         n.initialize(s)
256         self._configprops_cache = n.run('config-manager list properties')