X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fcloudtaf.git;a=blobdiff_plain;f=libraries%2Fcluster%2Fclusterverifier.py;fp=libraries%2Fcluster%2Fclusterverifier.py;h=0eb44cef36778d65a866755011f215cd0ff2b189;hp=0000000000000000000000000000000000000000;hb=d448b9388fd9cb3732e35996b98f493a5a5921d4;hpb=07c5f13d2429236a603c867e09c4cc3b42e75826 diff --git a/libraries/cluster/clusterverifier.py b/libraries/cluster/clusterverifier.py new file mode 100644 index 0000000..0eb44ce --- /dev/null +++ b/libraries/cluster/clusterverifier.py @@ -0,0 +1,366 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import itertools +from collections import namedtuple +import pytest +import six +import mock +from crl.remotesession.remotesession import RemoteSession +from hostcli import HostCli +from .cluster import ( + MgmtTarget, + Cluster, + ClusterError) +from .testutils.profiles import ( + MasterProfile, + WorkerProfile, + StorageProfile, + ManagementProfile, + BaseProfile) +from .testutils.ippool import IPPool +from .testutils.host import ( + MasterGenerator, + WorkerGenerator, + DpdkWorkerGenerator) +from .metasingleton import MetaSingleton + + +class ClusterMocks(namedtuple('ClusterMocks', ['remotesession', + 'hostcli', + 'envcreator', + 'usermanager'])): + pass + + +@six.add_metaclass(abc.ABCMeta) +class ClusterVerifierBase(object): + + _mgmt_target = MgmtTarget(host='host', + user='user', + password='password') + + _sudoshelldicts = [{'shellname': 'BashShell', 'cmd': 'sudo bash'}] + + def __init__(self, cluster_mocks): + self._mocks = cluster_mocks + self._ippool = IPPool() + self._hosts = [h for h in self._hosts_gen()] + self._cluster = None + MetaSingleton.clear(Cluster) + + @property + def cluster(self): + if self._cluster is None: + self._setup_cluster() + return self._cluster + + def _setup_cluster(self): + self._cluster = self._create_cluster() + + def verify_get_host(self): + for h in self._hosts: + actual_host = self.cluster.get_host(h.name) + assert actual_host.shelldicts == self._expected_shelldicts(h), ( + 'expected: {}, got: {}'.format( + self._expected_shelldicts(h), + actual_host.shelldicts)) + assert actual_host.network_domain == h.expected_network_domain, ( + 'expected: {expected}, got: {actual}'.format( + expected=h.expected_network_domain, + actual=actual_host.network_domain)) + assert actual_host.is_dpdk == h.expected_is_dpdk() + + def verify_master_external_ip(self): + for master in self._masters: + actual_ip = self.cluster.get_host(master.name).external_ip + assert actual_ip == master.external_ip, ( + 'Expected external_ip : {expected}, actual: {actual}'.format( + expected=master.external_ip, + actual=actual_ip)) + + @property + def _masters(self): + for h in self._hosts: + if MasterProfile in h.service_profiles: + yield h + + def verify_get_hosts_with_profiles(self): + for profs, eh in self._expected_restricted_profs.items(): + expected_hosts = sorted(eh) + hosts = self.cluster.get_hosts_with_profiles(*profs) + assert hosts == expected_hosts, ( + 'Expected hosts: {e}, got: {g}'.format(e=expected_hosts, g=hosts)) + + def verify_hosts_containing(self): + for p in self._profs_combinations(): + hosts = set(self.cluster.get_hosts_containing(*p)) + expected_hosts = set(self._get_expected_hosts_containing(set(p))) + assert set(hosts) == expected_hosts, ( + 'Expected hosts: {e}, got: {g}'.format(e=expected_hosts, g=hosts)) + + def verify_initialize_remotesession(self): + mock_remotesession = mock.create_autospec(RemoteSession).return_value + self.cluster.initialize_remotesession(mock_remotesession) + self._verify_initialize_mock_calls(mock_remotesession) + + def _verify_initialize_mock_calls(self, mock_remotesession): + mock_remotesession.set_envcreator.assert_called_once_with( + self._mocks.envcreator.return_value) + self._verify_normal_and_sudo_mgmt(mock_remotesession) + self._verify_remotescript_default(mock_remotesession) + for h in self.cluster.get_hosts(): + self._verify_normal_and_sudo_host(h, mock_remotesession) + + def _verify_normal_and_sudo_mgmt(self, mock_remotesession): + mgmtshelldicts = self.cluster.get_mgmt_shelldicts() + self._should_be_in_set_runner(mock.call(mgmtshelldicts), + mock_remotesession=mock_remotesession) + sudodicts = mgmtshelldicts + self._sudoshelldicts + self._should_be_in_set_runner(mock.call(shelldicts=sudodicts, + name='sudo-default'), + mock_remotesession=mock_remotesession) + + def _verify_remotescript_default(self, mock_remotesession): + self._should_be_in_set_target(mock.call(host=self._mgmt_target.host, + username=self._mgmt_target.user, + password=self._mgmt_target.password, + name='remotescript-default'), + mock_remotesession=mock_remotesession) + + def _should_be_in_set_target(self, call, mock_remotesession): + set_target_calls = self._get_set_target_calls(mock_remotesession) + assert call in set_target_calls, ( + '{call} is not in {set_target_calls}'.format( + call=call, + set_target_calls=set_target_calls)) + + @staticmethod + def _get_set_target_calls(mock_remotesession): + return mock_remotesession.set_target.mock_calls + + def _verify_normal_and_sudo_host(self, host, mock_remotesession): + self._should_be_in_set_runner(mock.call(shelldicts=host.shelldicts, + name=host.name), + mock_remotesession=mock_remotesession) + sudodicts = host.shelldicts + self._sudoshelldicts + self._should_be_in_set_runner(mock.call(shelldicts=sudodicts, + name='sudo-{}'.format(host.name)), + mock_remotesession=mock_remotesession) + + def verify_create_remotesession(self): + self._verify_initialize_mock_calls(self.cluster.create_remotesession()) + + def _should_be_in_set_runner(self, call, mock_remotesession): + set_runner_target_calls = self._get_set_runner_target_calls(mock_remotesession) + assert call in set_runner_target_calls, ( + '{call} is not in {set_runner_target_calls}'.format( + call=call, + set_runner_target_calls=set_runner_target_calls)) + + def verify_initialize_hostcli(self): + mock_hostcli = mock.create_autospec(HostCli) + self.cluster.initialize_hostcli(mock_hostcli) + mock_hostcli.initialize.assert_called_once_with( + self._mocks.remotesession.return_value) + + def verify_create_hostcli(self): + assert self.cluster.create_hostcli() == self._mocks.hostcli.return_value + assert self._mocks.hostcli.return_value.initialize.mock_calls == [ + mock.call(self._mocks.remotesession.return_value) for _ in range(2)] + + def verify_create_user_with_roles(self): + roles = ['role1', 'role2'] + create_user = self._mocks.usermanager.return_value.create_user_with_roles + assert self.cluster.create_user_with_roles(*roles) == create_user.return_value + self._mocks.usermanager.assert_called_once_with( + hostcli_factory=self._cluster.create_hostcli) + create_user.assert_called_once_with(*roles) + + def verify_delete_users(self): + self.cluster.delete_users() + self._mocks.usermanager.assert_called_once_with( + hostcli_factory=self._cluster.create_hostcli) + self._mocks.usermanager.return_value.delete_users.assert_called_once_with() + + def verify_envcreator(self): + self._setup_cluster() + self._mocks.envcreator.assert_called_once_with( + remotesession_factory=self.cluster.create_remotesession, + usermanager=self._mocks.usermanager.return_value) + + @staticmethod + def _get_set_runner_target_calls(mock_remotesession): + return mock_remotesession.set_runner_target.mock_calls + + def verify_cluster_config_caching(self): + self._create_cluster() + cluster = Cluster() + self._setup_cluster_and_verify(cluster) + + def verify_mgmt_shelldicts(self): + assert self.cluster.get_mgmt_shelldicts() == [self._mgmt_target.asdict()] + + def verify_is_dpdk(self): + assert self.cluster.is_dpdk() == self._expected_is_dpdk + + def verify_get_hosts_with_dpdk(self): + assert self.cluster.get_hosts_with_dpdk() == sorted(self._expected_hosts_with_dpdk()) + + def _expected_hosts_with_dpdk(self): + for h in self._hosts: + if h.expected_is_dpdk(): + yield h.name + + @property + def _expected_is_dpdk(self): + for h in self._hosts: + if h.expected_is_dpdk(): + return True + + return False + + @abc.abstractmethod + def _hosts_gen(self): + """Return generator of :class:`.testutils.Host` instances.""" + + def _create_cluster(self): + c = Cluster() + c.clear_cache() + self._setup_cluster_and_verify(c) + return c + + def _setup_cluster_and_verify(self, cluster): + self._mocks.hostcli.return_value.run.return_value = self._configprops + cluster.set_profiles(master=str(MasterProfile()), worker=str(WorkerProfile())) + cluster.initialize(**self._mgmt_target.asdict()) + self._verify_after_initialize(cluster) + + def _verify_after_initialize(self, cluster): + assert len(self._hosts) == len(cluster.get_hosts()), ( + len(self._hosts), len(cluster.get_hosts())) + + hostcli = self._mocks.hostcli.return_value + hostcli.initialize.assert_called_once_with( + self._mocks.remotesession.return_value) + hostcli.run.assert_called_once_with('config-manager list properties') + + def _get_expected_hosts_containing(self, profs): + hosts = set() + for p, h in self._expected_profs.items(): + if profs.issubset(p): + hosts = hosts.union(set(h)) + return hosts + + @property + def _expected_restricted_profs(self): + def profile_filter(prof): + return prof in [MasterProfile, WorkerProfile, StorageProfile] + + return self._get_expected_profs_for_filter(profile_filter) + + @property + def _expected_profs(self): + return self._get_expected_profs_for_filter(lambda prof: True) + + def _get_expected_profs_for_filter(self, profile_filter): + p = {} + for host in self._hosts: + key = frozenset([str(s()) for s in host.service_profiles if profile_filter(s)]) + if key not in p: + p[key] = [] + p[key].append(host.name) + return p + + @staticmethod + def _profs_combinations(): + profs = [str(p()) for p in [MasterProfile, + WorkerProfile, + BaseProfile, + ManagementProfile]] + for r in range(1, len(profs) + 1): + for p in itertools.combinations(profs, r): + yield p + + def _expected_shelldicts(self, host): + if MasterProfile in host.service_profiles: + return [{'host': host.external_ip, + 'user': self._mgmt_target.user, + 'password': self._mgmt_target.password}] + return [self._mgmt_target.asdict(), + {'host': host.internal_ip, + 'user': self._mgmt_target.user, + 'password': self._mgmt_target.password}] + + @property + def _configprops(self): + return list(self._hosts_config_gen()) + [self._get_hosts_network_profiles()] + + def _hosts_config_gen(self): + for h in self._hosts: + yield {'property': '{host}.networking'.format(host=h.name), + 'value': h.networking} + + yield {'property': 'cloud.hosts', + 'value': {h.name: h.host_dict for h in self._hosts}} + + def _get_hosts_network_profiles(self): + return {'property': 'cloud.network_profiles', + 'value': self._get_network_profile_details()} + + def _get_network_profile_details(self): + d = {} + for h in self._hosts: + d.update(h.network_profile_details) + return d + + @property + def _master_gen(self): + return MasterGenerator(self._ippool).gen + + @property + def _worker_gen(self): + return WorkerGenerator(self._ippool).gen + + +class Type1Verifier(ClusterVerifierBase): + # pylint: disable=not-callable + def _hosts_gen(self): + return itertools.chain(self._master_gen(3), + self._worker_gen(2)) + + +class Type2Verifier(ClusterVerifierBase): + + def _hosts_gen(self): + return itertools.chain(self._master_gen(3), + self._dpdk_worker_gen(2)) + + @property + def _dpdk_worker_gen(self): + return DpdkWorkerGenerator(self._ippool).gen + + +class CorruptedVerifier(Type1Verifier): + + def _hosts_config_gen(self): + yield {'property': 'not-relevant', + 'value': 'not-relevant'} + + def verify_corrupted_raises(self): + with pytest.raises(ClusterError) as exinfo: + self.cluster.get_host('somename') + + assert 'Property not found' in str(exinfo.value)