robot tcs, test charts, robot container added
[ta/cloudtaf.git] / cluster / clusterverifier.py
1 # Copyright 2019 Nokia
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import abc
16 import itertools
17 from collections import namedtuple
18 import pytest
19 import six
20 import mock
21 from crl.remotesession.remotesession import RemoteSession
22 from hostcli import HostCli
23 from .cluster import (
24     MgmtTarget,
25     Cluster,
26     ClusterError)
27 from .testutils.profiles import (
28     MasterProfile,
29     WorkerProfile,
30     StorageProfile,
31     ManagementProfile,
32     BaseProfile)
33 from .testutils.ippool import IPPool
34 from .testutils.host import (
35     MasterGenerator,
36     WorkerGenerator,
37     DpdkWorkerGenerator)
38 from .metasingleton import MetaSingleton
39
40
41 class ClusterMocks(namedtuple('ClusterMocks', ['remotesession',
42                                                'hostcli',
43                                                'envcreator',
44                                                'usermanager'])):
45     pass
46
47
48 @six.add_metaclass(abc.ABCMeta)
49 class ClusterVerifierBase(object):
50
51     _mgmt_target = MgmtTarget(host='host',
52                               user='user',
53                               password='password')
54
55     _sudoshelldicts = [{'shellname': 'BashShell', 'cmd': 'sudo bash'}]
56
57     def __init__(self, cluster_mocks):
58         self._mocks = cluster_mocks
59         self._ippool = IPPool()
60         self._hosts = [h for h in self._hosts_gen()]
61         self._cluster = None
62         MetaSingleton.clear(Cluster)
63
64     @property
65     def cluster(self):
66         if self._cluster is None:
67             self._setup_cluster()
68         return self._cluster
69
70     def _setup_cluster(self):
71         self._cluster = self._create_cluster()
72
73     def verify_get_host(self):
74         for h in self._hosts:
75             actual_host = self.cluster.get_host(h.name)
76             assert actual_host.shelldicts == self._expected_shelldicts(h), (
77                 'expected: {}, got: {}'.format(
78                     self._expected_shelldicts(h),
79                     actual_host.shelldicts))
80             assert actual_host.network_domain == h.expected_network_domain, (
81                 'expected: {expected}, got: {actual}'.format(
82                     expected=h.expected_network_domain,
83                     actual=actual_host.network_domain))
84             assert actual_host.is_dpdk == h.expected_is_dpdk()
85
86     def verify_master_external_ip(self):
87         for master in self._masters:
88             actual_ip = self.cluster.get_host(master.name).external_ip
89             assert actual_ip == master.external_ip, (
90                 'Expected external_ip : {expected}, actual: {actual}'.format(
91                     expected=master.external_ip,
92                     actual=actual_ip))
93
94     @property
95     def _masters(self):
96         for h in self._hosts:
97             if MasterProfile in h.service_profiles:
98                 yield h
99
100     def verify_get_hosts_with_profiles(self):
101         for profs, eh in self._expected_restricted_profs.items():
102             expected_hosts = sorted(eh)
103             hosts = self.cluster.get_hosts_with_profiles(*profs)
104             assert hosts == expected_hosts, (
105                 'Expected hosts: {e}, got: {g}'.format(e=expected_hosts, g=hosts))
106
107     def verify_hosts_containing(self):
108         for p in self._profs_combinations():
109             hosts = set(self.cluster.get_hosts_containing(*p))
110             expected_hosts = set(self._get_expected_hosts_containing(set(p)))
111             assert set(hosts) == expected_hosts, (
112                 'Expected hosts: {e}, got: {g}'.format(e=expected_hosts, g=hosts))
113
114     def verify_initialize_remotesession(self):
115         mock_remotesession = mock.create_autospec(RemoteSession).return_value
116         self.cluster.initialize_remotesession(mock_remotesession)
117         self._verify_initialize_mock_calls(mock_remotesession)
118
119     def _verify_initialize_mock_calls(self, mock_remotesession):
120         mock_remotesession.set_envcreator.assert_called_once_with(
121             self._mocks.envcreator.return_value)
122         self._verify_normal_and_sudo_mgmt(mock_remotesession)
123         self._verify_remotescript_default(mock_remotesession)
124         for h in self.cluster.get_hosts():
125             self._verify_normal_and_sudo_host(h, mock_remotesession)
126
127     def _verify_normal_and_sudo_mgmt(self, mock_remotesession):
128         mgmtshelldicts = self.cluster.get_mgmt_shelldicts()
129         self._should_be_in_set_runner(mock.call(mgmtshelldicts),
130                                       mock_remotesession=mock_remotesession)
131         sudodicts = mgmtshelldicts + self._sudoshelldicts
132         self._should_be_in_set_runner(mock.call(shelldicts=sudodicts,
133                                                 name='sudo-default'),
134                                       mock_remotesession=mock_remotesession)
135
136     def _verify_remotescript_default(self, mock_remotesession):
137         self._should_be_in_set_target(mock.call(host=self._mgmt_target.host,
138                                                 username=self._mgmt_target.user,
139                                                 password=self._mgmt_target.password,
140                                                 name='remotescript-default'),
141                                       mock_remotesession=mock_remotesession)
142
143     def _should_be_in_set_target(self, call, mock_remotesession):
144         set_target_calls = self._get_set_target_calls(mock_remotesession)
145         assert call in set_target_calls, (
146             '{call} is not in {set_target_calls}'.format(
147                 call=call,
148                 set_target_calls=set_target_calls))
149
150     @staticmethod
151     def _get_set_target_calls(mock_remotesession):
152         return mock_remotesession.set_target.mock_calls
153
154     def _verify_normal_and_sudo_host(self, host, mock_remotesession):
155         self._should_be_in_set_runner(mock.call(shelldicts=host.shelldicts,
156                                                 name=host.name),
157                                       mock_remotesession=mock_remotesession)
158         sudodicts = host.shelldicts + self._sudoshelldicts
159         self._should_be_in_set_runner(mock.call(shelldicts=sudodicts,
160                                                 name='sudo-{}'.format(host.name)),
161                                       mock_remotesession=mock_remotesession)
162
163     def verify_create_remotesession(self):
164         self._verify_initialize_mock_calls(self.cluster.create_remotesession())
165
166     def _should_be_in_set_runner(self, call, mock_remotesession):
167         set_runner_target_calls = self._get_set_runner_target_calls(mock_remotesession)
168         assert call in set_runner_target_calls, (
169             '{call} is not in {set_runner_target_calls}'.format(
170                 call=call,
171                 set_runner_target_calls=set_runner_target_calls))
172
173     def verify_initialize_hostcli(self):
174         mock_hostcli = mock.create_autospec(HostCli)
175         self.cluster.initialize_hostcli(mock_hostcli)
176         mock_hostcli.initialize.assert_called_once_with(
177             self._mocks.remotesession.return_value)
178
179     def verify_create_hostcli(self):
180         assert self.cluster.create_hostcli() == self._mocks.hostcli.return_value
181         assert self._mocks.hostcli.return_value.initialize.mock_calls == [
182             mock.call(self._mocks.remotesession.return_value) for _ in range(2)]
183
184     def verify_create_user_with_roles(self):
185         roles = ['role1', 'role2']
186         create_user = self._mocks.usermanager.return_value.create_user_with_roles
187         assert self.cluster.create_user_with_roles(*roles) == create_user.return_value
188         self._mocks.usermanager.assert_called_once_with(
189             hostcli_factory=self._cluster.create_hostcli)
190         create_user.assert_called_once_with(*roles)
191
192     def verify_delete_users(self):
193         self.cluster.delete_users()
194         self._mocks.usermanager.assert_called_once_with(
195             hostcli_factory=self._cluster.create_hostcli)
196         self._mocks.usermanager.return_value.delete_users.assert_called_once_with()
197
198     def verify_envcreator(self):
199         self._setup_cluster()
200         self._mocks.envcreator.assert_called_once_with(
201             remotesession_factory=self.cluster.create_remotesession,
202             usermanager=self._mocks.usermanager.return_value)
203
204     @staticmethod
205     def _get_set_runner_target_calls(mock_remotesession):
206         return mock_remotesession.set_runner_target.mock_calls
207
208     def verify_cluster_config_caching(self):
209         self._create_cluster()
210         cluster = Cluster()
211         self._setup_cluster_and_verify(cluster)
212
213     def verify_mgmt_shelldicts(self):
214         assert self.cluster.get_mgmt_shelldicts() == [self._mgmt_target.asdict()]
215
216     def verify_is_dpdk(self):
217         assert self.cluster.is_dpdk() == self._expected_is_dpdk
218
219     def verify_get_hosts_with_dpdk(self):
220         assert self.cluster.get_hosts_with_dpdk() == sorted(self._expected_hosts_with_dpdk())
221
222     def _expected_hosts_with_dpdk(self):
223         for h in self._hosts:
224             if h.expected_is_dpdk():
225                 yield h.name
226
227     @property
228     def _expected_is_dpdk(self):
229         for h in self._hosts:
230             if h.expected_is_dpdk():
231                 return True
232
233         return False
234
235     @abc.abstractmethod
236     def _hosts_gen(self):
237         """Return generator of :class:`.testutils.Host` instances."""
238
239     def _create_cluster(self):
240         c = Cluster()
241         c.clear_cache()
242         self._setup_cluster_and_verify(c)
243         return c
244
245     def _setup_cluster_and_verify(self, cluster):
246         self._mocks.hostcli.return_value.run.return_value = self._configprops
247         cluster.set_profiles(master=str(MasterProfile()), worker=str(WorkerProfile()))
248         cluster.initialize(**self._mgmt_target.asdict())
249         self._verify_after_initialize(cluster)
250
251     def _verify_after_initialize(self, cluster):
252         assert len(self._hosts) == len(cluster.get_hosts()), (
253             len(self._hosts), len(cluster.get_hosts()))
254
255         hostcli = self._mocks.hostcli.return_value
256         hostcli.initialize.assert_called_once_with(
257             self._mocks.remotesession.return_value)
258         hostcli.run.assert_called_once_with('config-manager list properties')
259
260     def _get_expected_hosts_containing(self, profs):
261         hosts = set()
262         for p, h in self._expected_profs.items():
263             if profs.issubset(p):
264                 hosts = hosts.union(set(h))
265         return hosts
266
267     @property
268     def _expected_restricted_profs(self):
269         def profile_filter(prof):
270             return prof in [MasterProfile, WorkerProfile, StorageProfile]
271
272         return self._get_expected_profs_for_filter(profile_filter)
273
274     @property
275     def _expected_profs(self):
276         return self._get_expected_profs_for_filter(lambda prof: True)
277
278     def _get_expected_profs_for_filter(self, profile_filter):
279         p = {}
280         for host in self._hosts:
281             key = frozenset([str(s()) for s in host.service_profiles if profile_filter(s)])
282             if key not in p:
283                 p[key] = []
284             p[key].append(host.name)
285         return p
286
287     @staticmethod
288     def _profs_combinations():
289         profs = [str(p()) for p in [MasterProfile,
290                                     WorkerProfile,
291                                     BaseProfile,
292                                     ManagementProfile]]
293         for r in range(1, len(profs) + 1):
294             for p in itertools.combinations(profs, r):
295                 yield p
296
297     def _expected_shelldicts(self, host):
298         if MasterProfile in host.service_profiles:
299             return [{'host': host.external_ip,
300                      'user': self._mgmt_target.user,
301                      'password': self._mgmt_target.password}]
302         return [self._mgmt_target.asdict(),
303                 {'host': host.internal_ip,
304                  'user': self._mgmt_target.user,
305                  'password': self._mgmt_target.password}]
306
307     @property
308     def _configprops(self):
309         return list(self._hosts_config_gen()) + [self._get_hosts_network_profiles()]
310
311     def _hosts_config_gen(self):
312         for h in self._hosts:
313             yield {'property': '{host}.networking'.format(host=h.name),
314                    'value': h.networking}
315
316         yield {'property': 'cloud.hosts',
317                'value': {h.name: h.host_dict for h in self._hosts}}
318
319     def _get_hosts_network_profiles(self):
320         return {'property': 'cloud.network_profiles',
321                 'value': self._get_network_profile_details()}
322
323     def _get_network_profile_details(self):
324         d = {}
325         for h in self._hosts:
326             d.update(h.network_profile_details)
327         return d
328
329     @property
330     def _master_gen(self):
331         return MasterGenerator(self._ippool).gen
332
333     @property
334     def _worker_gen(self):
335         return WorkerGenerator(self._ippool).gen
336
337
338 class Type1Verifier(ClusterVerifierBase):
339     # pylint: disable=not-callable
340     def _hosts_gen(self):
341         return itertools.chain(self._master_gen(3),
342                                self._worker_gen(2))
343
344
345 class Type2Verifier(ClusterVerifierBase):
346
347     def _hosts_gen(self):
348         return itertools.chain(self._master_gen(3),
349                                self._dpdk_worker_gen(2))
350
351     @property
352     def _dpdk_worker_gen(self):
353         return DpdkWorkerGenerator(self._ippool).gen
354
355
356 class CorruptedVerifier(Type1Verifier):
357
358     def _hosts_config_gen(self):
359         yield {'property': 'not-relevant',
360                'value': 'not-relevant'}
361
362     def verify_corrupted_raises(self):
363         with pytest.raises(ClusterError) as exinfo:
364             self.cluster.get_host('somename')
365
366         assert 'Property not found' in str(exinfo.value)