From d448b9388fd9cb3732e35996b98f493a5a5921d4 Mon Sep 17 00:00:00 2001 From: "Huovinen, Petri Juhani" Date: Wed, 13 Mar 2019 13:57:05 +0200 Subject: [PATCH] Add cloudtaf framework Add cloudtaf test automation framework with smoke tests. Change-Id: I472abab68974d67ab718997db10fbde6916f3663 --- .gitignore | 16 + .pylintrc | 406 +++++++++++++++++++++ Dockerfile | 31 ++ LICENSE | 202 ++++++++++ README.rst | 124 +++++++ libraries/cluster/__init__.py | 13 + libraries/cluster/cluster.py | 256 +++++++++++++ libraries/cluster/clusterverifier.py | 366 +++++++++++++++++++ libraries/cluster/conftest.py | 84 +++++ libraries/cluster/envcreator.py | 92 +++++ libraries/cluster/envcreatorverifier.py | 139 +++++++ libraries/cluster/hosts.py | 152 ++++++++ libraries/cluster/metasingleton.py | 28 ++ libraries/cluster/test_cluster.py | 91 +++++ libraries/cluster/test_envcreator.py | 29 ++ libraries/cluster/test_usergen.py | 32 ++ libraries/cluster/test_usermanager.py | 49 +++ libraries/cluster/testutils/__init__.py | 13 + libraries/cluster/testutils/fakehostcli.py | 330 +++++++++++++++++ libraries/cluster/testutils/host.py | 162 ++++++++ libraries/cluster/testutils/ippool.py | 32 ++ libraries/cluster/testutils/profiles.py | 38 ++ libraries/cluster/testutils/user_config.yaml | 16 + libraries/cluster/usergen.py | 39 ++ libraries/cluster/usermanager.py | 178 +++++++++ libraries/cluster/userverifier.py | 160 ++++++++ libraries/hostcli/__init__.py | 16 + libraries/hostcli/hostcli.py | 23 ++ libraries/hostcli/hostcliuser.py | 64 ++++ libraries/hostcli/test_hostcli.py | 37 ++ libraries/hostcli/test_hostcliuser.py | 128 +++++++ libraries/openstackcli/__init__.py | 18 + libraries/openstackcli/cliwrapperbase.py | 101 +++++ libraries/openstackcli/cloudcli.py | 81 ++++ libraries/openstackcli/conftest.py | 67 ++++ libraries/openstackcli/envcli.py | 40 ++ libraries/openstackcli/openstackcli.py | 404 ++++++++++++++++++++ .../service_show_neutron_expected_output.txt | 7 + libraries/openstackcli/test_openstack.py | 150 ++++++++ libraries/pytestremotesession/__init__.py | 0 libraries/pytestremotesession/fixtures.py | 29 ++ libraries/setup.py | 26 ++ requirements-minimal.txt | 3 + requirements.txt | 21 ++ resources/ssh.robot | 69 ++++ rfcli-docker | 55 +++ rfcli-docker-build | 23 ++ rfcli-tox-umask | 24 ++ targets/README.rst | 49 +++ testcases/__init__.robot | 32 ++ testcases/smoke-tests/smoke-tests.robot | 115 ++++++ tox.ini | 92 +++++ 52 files changed, 4752 insertions(+) create mode 100644 .gitignore create mode 100644 .pylintrc create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.rst create mode 100644 libraries/cluster/__init__.py create mode 100644 libraries/cluster/cluster.py create mode 100644 libraries/cluster/clusterverifier.py create mode 100644 libraries/cluster/conftest.py create mode 100644 libraries/cluster/envcreator.py create mode 100644 libraries/cluster/envcreatorverifier.py create mode 100644 libraries/cluster/hosts.py create mode 100644 libraries/cluster/metasingleton.py create mode 100644 libraries/cluster/test_cluster.py create mode 100644 libraries/cluster/test_envcreator.py create mode 100644 libraries/cluster/test_usergen.py create mode 100644 libraries/cluster/test_usermanager.py create mode 100644 libraries/cluster/testutils/__init__.py create mode 100644 libraries/cluster/testutils/fakehostcli.py create mode 100644 libraries/cluster/testutils/host.py create mode 100644 libraries/cluster/testutils/ippool.py create mode 100644 libraries/cluster/testutils/profiles.py create mode 100644 libraries/cluster/testutils/user_config.yaml create mode 100644 libraries/cluster/usergen.py create mode 100644 libraries/cluster/usermanager.py create mode 100644 libraries/cluster/userverifier.py create mode 100644 libraries/hostcli/__init__.py create mode 100644 libraries/hostcli/hostcli.py create mode 100644 libraries/hostcli/hostcliuser.py create mode 100644 libraries/hostcli/test_hostcli.py create mode 100644 libraries/hostcli/test_hostcliuser.py create mode 100644 libraries/openstackcli/__init__.py create mode 100644 libraries/openstackcli/cliwrapperbase.py create mode 100644 libraries/openstackcli/cloudcli.py create mode 100644 libraries/openstackcli/conftest.py create mode 100644 libraries/openstackcli/envcli.py create mode 100644 libraries/openstackcli/openstackcli.py create mode 100644 libraries/openstackcli/service_show_neutron_expected_output.txt create mode 100644 libraries/openstackcli/test_openstack.py create mode 100644 libraries/pytestremotesession/__init__.py create mode 100644 libraries/pytestremotesession/fixtures.py create mode 100644 libraries/setup.py create mode 100644 requirements-minimal.txt create mode 100644 requirements.txt create mode 100644 resources/ssh.robot create mode 100755 rfcli-docker create mode 100755 rfcli-docker-build create mode 100755 rfcli-tox-umask create mode 100644 targets/README.rst create mode 100644 testcases/__init__.robot create mode 100644 testcases/smoke-tests/smoke-tests.robot create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..07ba6ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +*.swp +*.swo +*.swn +*.tox/ +*.pyc +log.html +output.xml +report.html +.venv/ +libraries/cloudtaflibs.egg-info/ +.cache/ +.coverage.py27 +.coveragerc +coverage-html-py27/ +rfcli_output/ +targets/*.ini diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..fe5d705 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,406 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=build + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable= missing-docstring, locally-disabled + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,65}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,65}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_]|[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..74e0a0b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM fedora:26 + +RUN [ -z ${HTTP_PROXY:+x} ] || echo proxy=${HTTP_PROXY} >> /etc/dnf/dnf.conf + +RUN dnf install -y \ +python-pip \ +libffi-devel \ +gcc \ +openssl-libs \ +openssl \ +redhat-rpm-config \ +python-devel \ +openssl-devel \ +python3-tox \ +openssh-clients \ +git \ +iputils \ +graphviz diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..72549f4 --- /dev/null +++ b/README.rst @@ -0,0 +1,124 @@ +.. + Copyright 2019 Nokia + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Introduction +------------ + +Test cases are executed within a virtual environment in order to have all +python package dependencies installed. This virtual environment creation is +managed by tox. Furthermore tox execution can be optionally wrapped with a +docker environment to provide required operating system packages and +configuration to pull dependencies. + +Execution environment as simple diagram:: + + $ ./rfcli-docker == [ docker [ tox [ virtualenv [ rfcli -> robotframework ] ] ] ] + +You can find here more details about how to execute in your machine here +(windows specific, but it contains tips also if you have linux): +:ref:`windowsinstructions` + + +Writing robot test cases +------------------------ +Please use the following tags mentioned here: +:ref:`taggingpolicy` and also consider our test case design guidelines: +:ref:`designguidelines` + + +Execution examples +------------------ + +Docker + tox:: + + $ ./rfcli-docker -t path/to/target.ini -s smoke-tests testcases/ + +This is the most simple way to execute the cloudtaf2 tests. As docker is being +used the created python virtual environment as well as the resulting files will +be owned by the "root" and not the actual user. Additionally initial execution +will take some time to build the image. + +You may also run directly *rfcli* environment with *tox*:: + + $ tox -e rfcli -- -t path/to/target.ini -s smoke-tests testcases + + +Execution examples - tox only +----------------------------- + +This is more lightweight way to execute the tests but requires certain +operating system packages to be installed first. Please see the "Dockerfile" +for installed packages. + +Tox only - more lightweight for those who know what they are doing:: + + $ tox -e rfcli -- -t path/to/target.ini -s smoke-tests testcases/ + +Virtual environment +------------------- + +Python virtual environment is created with requirements.txt. All python +packages must be set to certain versions in order to execute tests the same way +with older builds as they have been initially executed. + +To update the frozen versions to the latest, execute:: + + $ tox -e freeze + +To add more packages, update the requirements-minimal.txt and execute:: + + $ tox -e freeze + +You can add specific version requirements to the requirements-minimal.txt +to make sure that they are used in the generated frozen requirements.txt. + +The recommendation is that the frozen requirements.txt file is not edited +directly but always via the requirements-minimal.txt changes and via this freeze +generation. + +In clear cases you can also update requirements.txt directly. However, +in this case, please make sure that:: + + $ tox --recreate -e check-requirements + +is succesful. This tool checks that the requirements-minimal.txt +is consistent with the requirements.txt and that all requirements in +requirements.txt can be really installed. +installed. + + +Rebuild docker image +-------------------- + +Rebuild docker image manually e.g. when changing the Dockerfile contents:: + + $ ./rfcli-docker-build + +Unit testing +------------ + +Unit tests can be executed with:: + + $ tox + +Running Docker behind a proxy +----------------------------- +The 'dnf install' -command requires a proxy setting when Docker is running +behind a proxy. + +The Dockerfile writes the proxy information to /etc/dnf/dnf.conf -file when +HTTP_PROXY argument is set as a --build-arg. For example:: + + # docker build --build-arg HTTP_PROXY=http://10.1.2.3:8080/ diff --git a/libraries/cluster/__init__.py b/libraries/cluster/__init__.py new file mode 100644 index 0000000..0cd2fda --- /dev/null +++ b/libraries/cluster/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/libraries/cluster/cluster.py b/libraries/cluster/cluster.py new file mode 100644 index 0000000..6b0f0f3 --- /dev/null +++ b/libraries/cluster/cluster.py @@ -0,0 +1,256 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import six +from crl.remotesession.remotesession import RemoteSession +from crl.interactivesessions.shells.bashshell import BashShell +from hostcli import HostCli +from .envcreator import EnvCreator +from .usermanager import UserManager +from .metasingleton import MetaSingleton +from .hosts import ( + MgmtTarget, + Master, + NonMaster, + Profiles, + HostConfig) + + +LOGGER = logging.getLogger(__name__) + + +class ClusterError(Exception): + pass + + +@six.add_metaclass(MetaSingleton) +class Cluster(object): + """Singleton container for NCIR cluster hosts, their service profiles and + access details. + """ + def __init__(self): + self._mgmt_target = None + self._hosts = None + self._configprops_cache = None + self._usermanager = UserManager(hostcli_factory=self.create_hostcli) + self._envcreator = EnvCreator(remotesession_factory=self.create_remotesession, + usermanager=self._usermanager) + + def clear_cache(self): + """Clears configprops cache.""" + self._configprops_cache = None + + def get_hosts(self): + """Return Host containers of cluster. + """ + return [h for _, h in self._hosts.items()] + + def initialize(self, host, user, password): + """Initialize Cluster with management VIP + + Arguments: + host: Management VIP IP address + user: Login username + password: password for the user + """ + self._mgmt_target = MgmtTarget(host=host, user=user, password=password) + self._hosts = {hc.name: self._create_host(hc) for hc in self._host_configs()} + + @staticmethod + def set_profiles(master, worker): + """Set *service_profile' names of *master* and *worker* nodes + """ + Profiles.set_profiles(master=master, worker=worker) + + def get_mgmt_shelldicts(self): + """Return management VIP shelldicts for RemoteRunner. + """ + return [self._mgmt_target.asdict()] + + def get_host(self, hostname): + """Get Host container for hostname.""" + return self._hosts[hostname] + + def get_hosts_with_profiles(self, *service_profiles): + """Get host names matching exactly *service_profiles*. + """ + return sorted(list(self._get_hosts_with_profs_gen(set(service_profiles)))) + + def get_hosts_containing(self, *service_profiles): + """Get host names containing all *service_profiles*. + """ + return sorted(list(self._get_hosts_containing_gen(set(service_profiles)))) + + def create_remotesession(self): + """Create initialized *RemoteSession* instance. + """ + r = RemoteSession() + self.initialize_remotesession(r) + return r + + def initialize_remotesession(self, remotesession): + """Initialize :class:`crl.remotesession.remotesession.RemoteSession` instance + with *shelldicts* and *name* of the hosts and sudo- via *set_runner_target*. + Initialize *default* target with *get_mgmt_shelldicts* return value. + The sudo- targets are terminals in target after executed roughly *sudo bash*. + """ + self._set_mgmt_targets(remotesession) + for host in self.get_hosts(): + remotesession.set_runner_target(shelldicts=host.shelldicts, + name=host.name) + remotesession.set_runner_target( + shelldicts=self._get_sudoshelldicts(host.shelldicts), + name='sudo-{}'.format(host.name)) + + remotesession.set_envcreator(self._envcreator) + + def _set_mgmt_targets(self, remotesession): + remotesession.set_runner_target(self.get_mgmt_shelldicts()) + remotesession.set_runner_target( + shelldicts=self._get_sudoshelldicts(self.get_mgmt_shelldicts()), + name='sudo-default') + remotesession.set_target(host=self._mgmt_target.host, + username=self._mgmt_target.user, + password=self._mgmt_target.password, + name='remotescript-default') + + @staticmethod + def _get_sudoshelldicts(shelldicts): + return shelldicts + [{'shellname': BashShell.__name__, + 'cmd': 'sudo bash'}] + + def create_hostcli(self): + """Create initialized *HostCli* instance. + """ + n = HostCli() + self.initialize_hostcli(n) + return n + + def initialize_hostcli(self, hostcli): + """Initialize :class:`crl.hostcli.HostCli` instance + (or :class:`crl.hostcli.OpenStack`) with + :class:`crl.remotesession.remotesession.RemoteSession` instance + initialized with :meth:`.initialize_remotesession`. + """ + hostcli.initialize(self.create_remotesession()) + + def create_user_with_roles(self, *roles): + """Create according to roles list. + + Special roles: + + all_roles: all roles in the system + no_roles: empty role list + + Return: + UserRecord of created user + """ + return self._usermanager.create_user_with_roles(*roles) + + def delete_users(self): + """Delete all users created by *Cluster*. + """ + self._usermanager.delete_users() + + def is_dpdk(self): + """Return *True* if *dpdk* is used in provider network interfaces. + More detail, *True* if and only if there is at least one host with + network profile providing *ovs-dpdk* type interface. + """ + for _, h in self._hosts.items(): + if h.is_dpdk: + return True + return False + + def get_hosts_with_dpdk(self): + """Returns list of hosts where *dpdk* is in use. + In more detail, return sorted list of host names in which there is + at least one network profile containing *dpdk* type interface. + """ + return sorted([h.name for _, h in self._hosts.items() if h.is_dpdk]) + + def _get_hosts_with_profs_gen(self, service_profiles): + def filt(host): + mask = Profiles().profiles_mask + return not (service_profiles ^ set(host.service_profiles)) & mask + + return self._filtered_hostnames(filt) + + def _get_hosts_containing_gen(self, service_profiles): + def filt(host): + return service_profiles.issubset(set(host.service_profiles)) + + return self._filtered_hostnames(filt) + + def _filtered_hostnames(self, filt): + for h in self.get_hosts(): + if filt(h): + yield h.name + + @staticmethod + def _create_host(host_config): + LOGGER.debug('host_config: %s', host_config) + master = Profiles().master + return (Master(host_config) + if master in host_config.service_profiles else + NonMaster(host_config)) + + def _host_configs(self): + LOGGER.debug('cloud_hosts: %s', self._cloud_hosts) + for hostname, v in self._cloud_hosts.items(): + yield HostConfig(name=hostname, + network_domain=v['network_domain'], + service_profiles=v['service_profiles'], + networking=self._get_networking(hostname), + mgmt_target=self._mgmt_target, + is_dpdk=self._is_host_dpdk(v)) + + def _is_host_dpdk(self, host_prop): + network_profiles = set(host_prop['network_profiles']) + return bool(network_profiles.intersection(set(self._dpdk_profiles()))) + + def _dpdk_profiles(self): + profiles = self._get_value_for_prop('cloud.network_profiles') + for prof_n, prof_v in profiles.items(): + ifaces = prof_v.get('provider_network_interfaces', {}) + for _, iface_v in ifaces.items(): + if iface_v['type'] == 'ovs-dpdk': + yield prof_n + + @property + def _cloud_hosts(self): + return self._get_value_for_prop('cloud.hosts') + + def _get_networking(self, hostname): + return self._get_value_for_prop('{}.networking'.format(hostname)) + + def _get_value_for_prop(self, prop): + for pv in self._configprops: + if pv['property'] == prop: + return pv['value'] + raise ClusterError('Property not found: {}'.format(prop)) + + @property + def _configprops(self): + if self._configprops_cache is None: + self._setup_configprops_cache() + return self._configprops_cache + + def _setup_configprops_cache(self): + s = RemoteSession() + s.set_runner_target(self.get_mgmt_shelldicts()) + n = HostCli() + n.initialize(s) + self._configprops_cache = n.run('config-manager list properties') diff --git a/libraries/cluster/clusterverifier.py b/libraries/cluster/clusterverifier.py new file mode 100644 index 0000000..0eb44ce --- /dev/null +++ b/libraries/cluster/clusterverifier.py @@ -0,0 +1,366 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import itertools +from collections import namedtuple +import pytest +import six +import mock +from crl.remotesession.remotesession import RemoteSession +from hostcli import HostCli +from .cluster import ( + MgmtTarget, + Cluster, + ClusterError) +from .testutils.profiles import ( + MasterProfile, + WorkerProfile, + StorageProfile, + ManagementProfile, + BaseProfile) +from .testutils.ippool import IPPool +from .testutils.host import ( + MasterGenerator, + WorkerGenerator, + DpdkWorkerGenerator) +from .metasingleton import MetaSingleton + + +class ClusterMocks(namedtuple('ClusterMocks', ['remotesession', + 'hostcli', + 'envcreator', + 'usermanager'])): + pass + + +@six.add_metaclass(abc.ABCMeta) +class ClusterVerifierBase(object): + + _mgmt_target = MgmtTarget(host='host', + user='user', + password='password') + + _sudoshelldicts = [{'shellname': 'BashShell', 'cmd': 'sudo bash'}] + + def __init__(self, cluster_mocks): + self._mocks = cluster_mocks + self._ippool = IPPool() + self._hosts = [h for h in self._hosts_gen()] + self._cluster = None + MetaSingleton.clear(Cluster) + + @property + def cluster(self): + if self._cluster is None: + self._setup_cluster() + return self._cluster + + def _setup_cluster(self): + self._cluster = self._create_cluster() + + def verify_get_host(self): + for h in self._hosts: + actual_host = self.cluster.get_host(h.name) + assert actual_host.shelldicts == self._expected_shelldicts(h), ( + 'expected: {}, got: {}'.format( + self._expected_shelldicts(h), + actual_host.shelldicts)) + assert actual_host.network_domain == h.expected_network_domain, ( + 'expected: {expected}, got: {actual}'.format( + expected=h.expected_network_domain, + actual=actual_host.network_domain)) + assert actual_host.is_dpdk == h.expected_is_dpdk() + + def verify_master_external_ip(self): + for master in self._masters: + actual_ip = self.cluster.get_host(master.name).external_ip + assert actual_ip == master.external_ip, ( + 'Expected external_ip : {expected}, actual: {actual}'.format( + expected=master.external_ip, + actual=actual_ip)) + + @property + def _masters(self): + for h in self._hosts: + if MasterProfile in h.service_profiles: + yield h + + def verify_get_hosts_with_profiles(self): + for profs, eh in self._expected_restricted_profs.items(): + expected_hosts = sorted(eh) + hosts = self.cluster.get_hosts_with_profiles(*profs) + assert hosts == expected_hosts, ( + 'Expected hosts: {e}, got: {g}'.format(e=expected_hosts, g=hosts)) + + def verify_hosts_containing(self): + for p in self._profs_combinations(): + hosts = set(self.cluster.get_hosts_containing(*p)) + expected_hosts = set(self._get_expected_hosts_containing(set(p))) + assert set(hosts) == expected_hosts, ( + 'Expected hosts: {e}, got: {g}'.format(e=expected_hosts, g=hosts)) + + def verify_initialize_remotesession(self): + mock_remotesession = mock.create_autospec(RemoteSession).return_value + self.cluster.initialize_remotesession(mock_remotesession) + self._verify_initialize_mock_calls(mock_remotesession) + + def _verify_initialize_mock_calls(self, mock_remotesession): + mock_remotesession.set_envcreator.assert_called_once_with( + self._mocks.envcreator.return_value) + self._verify_normal_and_sudo_mgmt(mock_remotesession) + self._verify_remotescript_default(mock_remotesession) + for h in self.cluster.get_hosts(): + self._verify_normal_and_sudo_host(h, mock_remotesession) + + def _verify_normal_and_sudo_mgmt(self, mock_remotesession): + mgmtshelldicts = self.cluster.get_mgmt_shelldicts() + self._should_be_in_set_runner(mock.call(mgmtshelldicts), + mock_remotesession=mock_remotesession) + sudodicts = mgmtshelldicts + self._sudoshelldicts + self._should_be_in_set_runner(mock.call(shelldicts=sudodicts, + name='sudo-default'), + mock_remotesession=mock_remotesession) + + def _verify_remotescript_default(self, mock_remotesession): + self._should_be_in_set_target(mock.call(host=self._mgmt_target.host, + username=self._mgmt_target.user, + password=self._mgmt_target.password, + name='remotescript-default'), + mock_remotesession=mock_remotesession) + + def _should_be_in_set_target(self, call, mock_remotesession): + set_target_calls = self._get_set_target_calls(mock_remotesession) + assert call in set_target_calls, ( + '{call} is not in {set_target_calls}'.format( + call=call, + set_target_calls=set_target_calls)) + + @staticmethod + def _get_set_target_calls(mock_remotesession): + return mock_remotesession.set_target.mock_calls + + def _verify_normal_and_sudo_host(self, host, mock_remotesession): + self._should_be_in_set_runner(mock.call(shelldicts=host.shelldicts, + name=host.name), + mock_remotesession=mock_remotesession) + sudodicts = host.shelldicts + self._sudoshelldicts + self._should_be_in_set_runner(mock.call(shelldicts=sudodicts, + name='sudo-{}'.format(host.name)), + mock_remotesession=mock_remotesession) + + def verify_create_remotesession(self): + self._verify_initialize_mock_calls(self.cluster.create_remotesession()) + + def _should_be_in_set_runner(self, call, mock_remotesession): + set_runner_target_calls = self._get_set_runner_target_calls(mock_remotesession) + assert call in set_runner_target_calls, ( + '{call} is not in {set_runner_target_calls}'.format( + call=call, + set_runner_target_calls=set_runner_target_calls)) + + def verify_initialize_hostcli(self): + mock_hostcli = mock.create_autospec(HostCli) + self.cluster.initialize_hostcli(mock_hostcli) + mock_hostcli.initialize.assert_called_once_with( + self._mocks.remotesession.return_value) + + def verify_create_hostcli(self): + assert self.cluster.create_hostcli() == self._mocks.hostcli.return_value + assert self._mocks.hostcli.return_value.initialize.mock_calls == [ + mock.call(self._mocks.remotesession.return_value) for _ in range(2)] + + def verify_create_user_with_roles(self): + roles = ['role1', 'role2'] + create_user = self._mocks.usermanager.return_value.create_user_with_roles + assert self.cluster.create_user_with_roles(*roles) == create_user.return_value + self._mocks.usermanager.assert_called_once_with( + hostcli_factory=self._cluster.create_hostcli) + create_user.assert_called_once_with(*roles) + + def verify_delete_users(self): + self.cluster.delete_users() + self._mocks.usermanager.assert_called_once_with( + hostcli_factory=self._cluster.create_hostcli) + self._mocks.usermanager.return_value.delete_users.assert_called_once_with() + + def verify_envcreator(self): + self._setup_cluster() + self._mocks.envcreator.assert_called_once_with( + remotesession_factory=self.cluster.create_remotesession, + usermanager=self._mocks.usermanager.return_value) + + @staticmethod + def _get_set_runner_target_calls(mock_remotesession): + return mock_remotesession.set_runner_target.mock_calls + + def verify_cluster_config_caching(self): + self._create_cluster() + cluster = Cluster() + self._setup_cluster_and_verify(cluster) + + def verify_mgmt_shelldicts(self): + assert self.cluster.get_mgmt_shelldicts() == [self._mgmt_target.asdict()] + + def verify_is_dpdk(self): + assert self.cluster.is_dpdk() == self._expected_is_dpdk + + def verify_get_hosts_with_dpdk(self): + assert self.cluster.get_hosts_with_dpdk() == sorted(self._expected_hosts_with_dpdk()) + + def _expected_hosts_with_dpdk(self): + for h in self._hosts: + if h.expected_is_dpdk(): + yield h.name + + @property + def _expected_is_dpdk(self): + for h in self._hosts: + if h.expected_is_dpdk(): + return True + + return False + + @abc.abstractmethod + def _hosts_gen(self): + """Return generator of :class:`.testutils.Host` instances.""" + + def _create_cluster(self): + c = Cluster() + c.clear_cache() + self._setup_cluster_and_verify(c) + return c + + def _setup_cluster_and_verify(self, cluster): + self._mocks.hostcli.return_value.run.return_value = self._configprops + cluster.set_profiles(master=str(MasterProfile()), worker=str(WorkerProfile())) + cluster.initialize(**self._mgmt_target.asdict()) + self._verify_after_initialize(cluster) + + def _verify_after_initialize(self, cluster): + assert len(self._hosts) == len(cluster.get_hosts()), ( + len(self._hosts), len(cluster.get_hosts())) + + hostcli = self._mocks.hostcli.return_value + hostcli.initialize.assert_called_once_with( + self._mocks.remotesession.return_value) + hostcli.run.assert_called_once_with('config-manager list properties') + + def _get_expected_hosts_containing(self, profs): + hosts = set() + for p, h in self._expected_profs.items(): + if profs.issubset(p): + hosts = hosts.union(set(h)) + return hosts + + @property + def _expected_restricted_profs(self): + def profile_filter(prof): + return prof in [MasterProfile, WorkerProfile, StorageProfile] + + return self._get_expected_profs_for_filter(profile_filter) + + @property + def _expected_profs(self): + return self._get_expected_profs_for_filter(lambda prof: True) + + def _get_expected_profs_for_filter(self, profile_filter): + p = {} + for host in self._hosts: + key = frozenset([str(s()) for s in host.service_profiles if profile_filter(s)]) + if key not in p: + p[key] = [] + p[key].append(host.name) + return p + + @staticmethod + def _profs_combinations(): + profs = [str(p()) for p in [MasterProfile, + WorkerProfile, + BaseProfile, + ManagementProfile]] + for r in range(1, len(profs) + 1): + for p in itertools.combinations(profs, r): + yield p + + def _expected_shelldicts(self, host): + if MasterProfile in host.service_profiles: + return [{'host': host.external_ip, + 'user': self._mgmt_target.user, + 'password': self._mgmt_target.password}] + return [self._mgmt_target.asdict(), + {'host': host.internal_ip, + 'user': self._mgmt_target.user, + 'password': self._mgmt_target.password}] + + @property + def _configprops(self): + return list(self._hosts_config_gen()) + [self._get_hosts_network_profiles()] + + def _hosts_config_gen(self): + for h in self._hosts: + yield {'property': '{host}.networking'.format(host=h.name), + 'value': h.networking} + + yield {'property': 'cloud.hosts', + 'value': {h.name: h.host_dict for h in self._hosts}} + + def _get_hosts_network_profiles(self): + return {'property': 'cloud.network_profiles', + 'value': self._get_network_profile_details()} + + def _get_network_profile_details(self): + d = {} + for h in self._hosts: + d.update(h.network_profile_details) + return d + + @property + def _master_gen(self): + return MasterGenerator(self._ippool).gen + + @property + def _worker_gen(self): + return WorkerGenerator(self._ippool).gen + + +class Type1Verifier(ClusterVerifierBase): + # pylint: disable=not-callable + def _hosts_gen(self): + return itertools.chain(self._master_gen(3), + self._worker_gen(2)) + + +class Type2Verifier(ClusterVerifierBase): + + def _hosts_gen(self): + return itertools.chain(self._master_gen(3), + self._dpdk_worker_gen(2)) + + @property + def _dpdk_worker_gen(self): + return DpdkWorkerGenerator(self._ippool).gen + + +class CorruptedVerifier(Type1Verifier): + + def _hosts_config_gen(self): + yield {'property': 'not-relevant', + 'value': 'not-relevant'} + + def verify_corrupted_raises(self): + with pytest.raises(ClusterError) as exinfo: + self.cluster.get_host('somename') + + assert 'Property not found' in str(exinfo.value) diff --git a/libraries/cluster/conftest.py b/libraries/cluster/conftest.py new file mode 100644 index 0000000..94d581d --- /dev/null +++ b/libraries/cluster/conftest.py @@ -0,0 +1,84 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name +import pytest +import mock +from crl.remotesession.remotesession import RemoteSession +from hostcli import HostCli +from .clusterverifier import ( + Type1Verifier, + Type2Verifier, + ClusterMocks) +from .userverifier import UserVerifier +from .envcreatorverifier import EnvCreatorVerifier +from . import cluster +from . import usermanager +from .envcreator import EnvCreator + + +@pytest.fixture(params=[Type1Verifier, + Type2Verifier]) +def clusterverifier(request, clustermocks): + return request.param(clustermocks) + + +@pytest.fixture +def clustermocks(mock_remotesession, mock_hostcli, mock_envcreator, mock_usermanager): + return ClusterMocks(remotesession=mock_remotesession, + hostcli=mock_hostcli, + envcreator=mock_envcreator, + usermanager=mock_usermanager) + + +@pytest.fixture +def mock_remotesession(): + with mock.patch.object(cluster, + 'RemoteSession', + mock.create_autospec(RemoteSession)) as p: + yield p + + +@pytest.fixture +def mock_hostcli(): + with mock.patch.object(cluster, + 'HostCli', + mock.create_autospec(HostCli)) as p: + yield p + + +@pytest.fixture +def mock_envcreator(): + with mock.patch.object(cluster, + 'EnvCreator', + mock.create_autospec(EnvCreator)) as p: + yield p + + +@pytest.fixture +def mock_usermanager(): + with mock.patch.object(cluster, + 'UserManager', + mock.create_autospec(usermanager.UserManager)) as p: + yield p + + +@pytest.fixture(params=['Role', 'Role-Name']) +def userverifier(request): + return UserVerifier(role_attr=request.param) + + +@pytest.fixture +def envcreatorverifier(): + return EnvCreatorVerifier() diff --git a/libraries/cluster/envcreator.py b/libraries/cluster/envcreator.py new file mode 100644 index 0000000..9ee0936 --- /dev/null +++ b/libraries/cluster/envcreator.py @@ -0,0 +1,92 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +import yaml +from crl.remotesession.envcreatorbase import EnvCreatorBase + + +class EnvCreator(EnvCreatorBase): + _openrc = 'openrc' + + def __init__(self, remotesession_factory, usermanager): + self._remotesession_factory = remotesession_factory + self._usermanager = usermanager + self._openrc_dict_cache = None + self._um_admin_creds_cache = None + + def create(self, target, envname): + openrc_dict = self._get_openrc_dict(target) + if envname.startswith('set_password:'): + self._set_password_and_update_dict(envname, openrc_dict) + elif envname == 'um_admin': + self._update_openrc_dict_um_admin(openrc_dict) + elif envname != 'admin': + self._create_user_and_update_dict(envname, openrc_dict=openrc_dict) + + return openrc_dict + + @property + def _um_admin_creds(self): + if self._um_admin_creds_cache is None: + self._setup_um_admin_creds_cache() + return self._um_admin_creds_cache + + def _set_password_and_update_dict(self, envname, openrc_dict): + roles = self._get_roles_for_envname(envname) + userrecord = self._usermanager.get_user_and_set_password(*roles) + self._update_openrc_dict(openrc_dict, userrecord) + + @staticmethod + def _get_roles_for_envname(envname): + r = envname.split(':')[1] + return [role.strip() for role in r.split(',')] if r else [] + + @staticmethod + def _update_openrc_dict(openrc_dict, userrecord): + openrc_dict.update({'OS_USERNAME': userrecord.username, + 'OS_PASSWORD': userrecord.password, + 'OS_TENANT_NAME': 'infrastructure', + 'OS_PROJECT_NAME': 'infrastructure'}) + + def _update_openrc_dict_um_admin(self, openrc_dict): + openrc_dict.update({'OS_USERNAME': self._um_admin_creds.username, + 'OS_PASSWORD': self._um_admin_creds.password, + 'OS_TENANT_NAME': 'infrastructure', + 'OS_PROJECT_NAME': 'infrastructure'}) + + def _create_user_and_update_dict(self, envname, openrc_dict): + roles = [role.strip() for role in envname.split(',')] + userrecord = self._usermanager.create_user_with_roles(*roles) + self._update_openrc_dict(openrc_dict, userrecord) + + def _get_openrc_dict(self, target): + remotesession = self._remotesession_factory() + if self._openrc_dict_cache is None: + self._openrc_dict_cache = ( + remotesession.get_source_update_env_dict(self._openrc, + target=target)) + return self._openrc_dict_cache.copy() + + def _setup_um_admin_creds_cache(self): + rem = self._remotesession_factory() + res = rem.execute_command_in_target('cat /etc/userconfig/user_config.yaml') + u = yaml.load(res.stdout, Loader=yaml.Loader) + users = u['users'] + self._um_admin_creds_cache = Credentials(username=users['initial_user_name'], + password=users['initial_user_password']) + + +class Credentials(namedtuple('Credentials', ['username', 'password'])): + pass diff --git a/libraries/cluster/envcreatorverifier.py b/libraries/cluster/envcreatorverifier.py new file mode 100644 index 0000000..9812d1d --- /dev/null +++ b/libraries/cluster/envcreatorverifier.py @@ -0,0 +1,139 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import mock +from crl.remotesession.remotesession import RemoteSession +from crl.interactivesessions.remoterunner import RunResult +from .usermanager import ( + UserManager, + UserRecord) +from .envcreator import EnvCreator + + +UPDATE_ENV_DICT = { + 'OS_ENDPOINT_TYPE': 'internalURL', + 'OS_USERNAME': 'admin', + 'OS_PASSWORD': 'password', + 'OS_TENANT_NAME': 'admin', + 'OS_PROJECT_NAME': 'admin', + 'OS_AUTH_URL': '192.168.1.2:5000/v3'} + +UM_ADMIN_ENV_DICT = UPDATE_ENV_DICT.copy() +UM_ADMIN_ENV_DICT.update({'OS_USERNAME': 'um_admin_username', + 'OS_PASSWORD': 'um_admin_password', + 'OS_TENANT_NAME': 'infrastructure', + 'OS_PROJECT_NAME': 'infrastructure'}) + +ROLES = ['role1', 'role2'] + +USERRECORD = UserRecord(uuid='uuid', + username='username', + password='password', + roles=ROLES) + +TARGET = 'target' + + +THISDIR = os.path.dirname(__file__) +USER_CONFIG_PATH = os.path.join(THISDIR, 'testutils', 'user_config.yaml') + + +class EnvCreatorVerifier(object): + def __init__(self): + self._mock_remotesession_factory = mock.create_autospec(RemoteSession) + self._mock_usermanager = mock.create_autospec(UserManager) + self._envcreator = EnvCreator( + remotesession_factory=self._mock_remotesession_factory, + usermanager=self._mock_usermanager) + self._setup_mocks() + + @property + def _envname(self): + return ', '.join(ROLES) + + def _setup_mocks(self): + self._setup_mock_remotesession() + self._setup_mock_usermanager() + + def _setup_mock_remotesession(self): + self._get_source_update_env_dict.return_value = UPDATE_ENV_DICT + self._execute_command_in_target.side_effect = self._execute_side_effect + + @property + def _get_source_update_env_dict(self): + return self._mock_remotesession_factory.return_value.get_source_update_env_dict + + @property + def _execute_command_in_target(self): + return self._mock_remotesession_factory.return_value.execute_command_in_target + + @staticmethod + def _execute_side_effect(command): + assert command == 'cat /etc/userconfig/user_config.yaml' + with open(USER_CONFIG_PATH) as f: + return RunResult(status=0, stdout=f.read(), stderr='') + + def _setup_mock_usermanager(self): + def mock_create_user_with_roles(*roles): + roles_list = list(roles) + assert list(roles_list) == ROLES, ( + 'Expected {expected!r}, actual {actual!r}'.format( + expected=ROLES, + actual=roles_list)) + return USERRECORD + + self._mock_usermanager.create_user_with_roles.side_effect = mock_create_user_with_roles + + def verify_create(self): + self._assert_target_dict(self._envcreator.create(target=TARGET, + envname=self._envname)) + self._verify_mock_calls() + + def verify_multiple_creates(self): + self.verify_create() + self._assert_target_dict(self._envcreator.create(target='target2', + envname=self._envname)) + self._get_source_update_env_dict.assert_called_once_with( + self._openrc, target=TARGET) + + def verify_create_admin(self): + assert self._envcreator.create(target=TARGET, envname='admin') == UPDATE_ENV_DICT + self._verify_mock_calls() + + def verify_create_um_admin(self): + assert self._envcreator.create(target=TARGET, envname='um_admin') == UM_ADMIN_ENV_DICT + + def _assert_target_dict(self, target_dict): + assert target_dict == self._expected_target_dict, ( + 'Expcted: {expected}, actual: {actual}'.format( + expected=self._expected_target_dict, + actual=target_dict)) + + @property + def _expected_target_dict(self): + d = UPDATE_ENV_DICT.copy() + d.update({'OS_USERNAME': USERRECORD.username, + 'OS_PASSWORD': USERRECORD.password, + 'OS_TENANT_NAME': 'infrastructure', + 'OS_PROJECT_NAME': 'infrastructure'}) + return d + + def _verify_mock_calls(self): + self._get_source_update_env_dict.assert_called_once_with( + self._openrc, target=TARGET) + + @property + def _openrc(self): + return 'openrc' diff --git a/libraries/cluster/hosts.py b/libraries/cluster/hosts.py new file mode 100644 index 0000000..917f730 --- /dev/null +++ b/libraries/cluster/hosts.py @@ -0,0 +1,152 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +from collections import namedtuple +import six + + +STORAGE = 'storage' + + +class Profiles(object): + _master = None + _worker = None + + @classmethod + def set_profiles(cls, master, worker): + cls._master = master + cls._worker = worker + + @property + def master(self): + return self._master + + @property + def worker(self): + return self._worker + + @property + def storage(self): + return STORAGE + + @property + def profiles_mask(self): + return set([self.master, self.worker, self.storage]) + + +class MgmtTarget(namedtuple('MgmtTarget', ['host', 'user', 'password'])): + """Container for the cloudtaf2 management VIP target attributes. + + Arguments: + host: IP address or FQDN of the host management VIP + user: Username, e.g. cloudadmin for login + password: Login password + + Example: + + Library cluster.cluster.MgmtTarget + ... host=1.2.3.4 + ... user=cloudadmin + ... password=good_password + """ + + def asdict(self): + return self._asdict() + + +@six.add_metaclass(abc.ABCMeta) +class HostBase(object): + """Container base for Host attributes. + + Attributes: + name: name of the host + service_profiles: list of service profiles like ['master', 'worker'] + shelldicts: list of dictionaries to RemoteSession.set_runner_target + """ + def __init__(self, host_config): + self._host_config = host_config + + @property + def is_dpdk(self): + """Return True if dpdk is used in provider network interfaces. + In more detail, is True if and only if one of the network profiles has + at least one interface with type *ovs-dpdk*. + """ + return self._host_config.is_dpdk + + @property + def name(self): + return self._host_config.name + + @property + def service_profiles(self): + return self._host_config.service_profiles + + @property + def network_domain(self): + return self._host_config.network_domain + + @abc.abstractproperty + def shelldicts(self): + """Return *shelldicts* for :class:`crl.remotesession.remotesession`. + """ + + @property + def _host_dict(self): + return {'host': self._infra['ip'], + 'user': self._host_config.mgmt_target.user, + 'password': self._host_config.mgmt_target.password} + + @property + def _infra(self): + return self._host_config.networking['infra_{}'.format(self._infra_type)] + + @abc.abstractproperty + def _infra_type(self): + """Return any infra type e.g. 'internal', 'external' etc. + """ + + +class Master(HostBase): + @property + def shelldicts(self): + return [self._host_dict] + + @property + def _infra_type(self): + return 'external' + + @property + def external_ip(self): + return self._infra['ip'] + + +class NonMaster(HostBase): + @property + def shelldicts(self): + return [self._host_config.mgmt_target.asdict(), self._host_dict] + + @property + def _infra_type(self): + return 'internal' + + +class HostConfig(namedtuple('HostConfig', ['name', + 'network_domain', + 'service_profiles', + 'networking', + 'mgmt_target', + 'is_dpdk'])): + pass diff --git a/libraries/cluster/metasingleton.py b/libraries/cluster/metasingleton.py new file mode 100644 index 0000000..54e832e --- /dev/null +++ b/libraries/cluster/metasingleton.py @@ -0,0 +1,28 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class MetaSingleton(type): + instance_attr = '__instance' + + def __call__(cls, *args, **kwargs): + if MetaSingleton.instance_attr not in cls.__dict__: + setattr(cls, + MetaSingleton.instance_attr, + super(MetaSingleton, cls).__call__(*args, **kwargs)) + return getattr(cls, MetaSingleton.instance_attr) + + def clear(cls): + if MetaSingleton.instance_attr in cls.__dict__: + delattr(cls, MetaSingleton.instance_attr) diff --git a/libraries/cluster/test_cluster.py b/libraries/cluster/test_cluster.py new file mode 100644 index 0000000..9b0edf0 --- /dev/null +++ b/libraries/cluster/test_cluster.py @@ -0,0 +1,91 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from crl.interactivesessions.shells.sudoshell import BashShell +from .clusterverifier import CorruptedVerifier +from .cluster import Cluster +from . import cluster + + +def test_get_host(clusterverifier): + clusterverifier.verify_get_host() + + +def test_master_external_ip(clusterverifier): + clusterverifier.verify_master_external_ip() + + +def test_get_hosts_with_profile(clusterverifier): + clusterverifier.verify_get_hosts_with_profiles() + + +def test_get_hosts_containing(clusterverifier): + clusterverifier.verify_hosts_containing() + + +def test_cluster_caching(clusterverifier): + clusterverifier.verify_cluster_config_caching() + + +def test_cluster_singleton(): + assert Cluster() == Cluster() + + +def test_cluster_mgmt_shelldicts(clusterverifier): + clusterverifier.verify_mgmt_shelldicts() + + +def test_get_host_raises(clustermocks): + c = CorruptedVerifier(clustermocks) + c.verify_corrupted_raises() + + +def test_create_remotesession(clusterverifier): + clusterverifier.verify_create_remotesession() + + +def test_initialize_remotesession(clusterverifier): + clusterverifier.verify_initialize_remotesession() + + +def test_create_hostcli(clusterverifier): + clusterverifier.verify_create_hostcli() + + +def test_initialize_hostcli(clusterverifier): + clusterverifier.verify_initialize_hostcli() + + +def test_create_user_with_roles(clusterverifier): + clusterverifier.verify_create_user_with_roles() + + +def test_delete_users(clusterverifier): + clusterverifier.verify_delete_users() + + +def test_envcreator_usage(clusterverifier): + clusterverifier.verify_envcreator() + + +def test_sudoshell_in_cluster(): + assert cluster.BashShell == BashShell + + +def test_is_dpdk(clusterverifier): + clusterverifier.verify_is_dpdk() + + +def test_get_hosts_with_dpdk(clusterverifier): + clusterverifier.verify_get_hosts_with_dpdk() diff --git a/libraries/cluster/test_envcreator.py b/libraries/cluster/test_envcreator.py new file mode 100644 index 0000000..de57d16 --- /dev/null +++ b/libraries/cluster/test_envcreator.py @@ -0,0 +1,29 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_create(envcreatorverifier): + envcreatorverifier.verify_create() + + +def test_create_multiple(envcreatorverifier): + envcreatorverifier.verify_multiple_creates() + + +def test_create_admin(envcreatorverifier): + envcreatorverifier.verify_create_admin() + + +def test_create_um_admin(envcreatorverifier): + envcreatorverifier.verify_create_um_admin() diff --git a/libraries/cluster/test_usergen.py b/libraries/cluster/test_usergen.py new file mode 100644 index 0000000..d1439f1 --- /dev/null +++ b/libraries/cluster/test_usergen.py @@ -0,0 +1,32 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .usergen import UserGen + + +def test_create_with_no_roles(): + u = UserGen(3) + assert u.create_username([]) == 'no_roles' + + +def test_create_with_all_roles(): + u = UserGen(3) + assert u.create_username(['role{}'.format(i) for i in range(3)]) == 'all_roles' + + +def test_create_with_some_roles(): + u = UserGen(3) + roles = ['role1', 'role0'] + assert u.create_username(roles) == 'role00' + assert u.create_username(roles) == 'role01' diff --git a/libraries/cluster/test_usermanager.py b/libraries/cluster/test_usermanager.py new file mode 100644 index 0000000..e545552 --- /dev/null +++ b/libraries/cluster/test_usermanager.py @@ -0,0 +1,49 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_create_user_with_roles(userverifier): + userverifier.verify_create_user_with_roles() + + +def test_delete_users(userverifier): + userverifier.verify_delete_users() + + +def test_corrupted_user_list(userverifier): + userverifier.verify_corrupted_user_list() + + +def test_user_with_roles_raises(userverifier): + userverifier.verify_user_with_roles_notexist() + + +def test_user_roles_duplicates(userverifier): + userverifier.verify_user_roles_duplicates() + + +def test_one_user_per_roles(userverifier): + userverifier.verify_one_user_per_roles() + + +def test_all_roles(userverifier): + userverifier.verify_all_roles() + + +def test_no_roles(userverifier): + userverifier.verify_no_roles() + + +def test_special_roles_raises(userverifier): + userverifier.verify_special_roles_raises() diff --git a/libraries/cluster/testutils/__init__.py b/libraries/cluster/testutils/__init__.py new file mode 100644 index 0000000..0cd2fda --- /dev/null +++ b/libraries/cluster/testutils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/libraries/cluster/testutils/fakehostcli.py b/libraries/cluster/testutils/fakehostcli.py new file mode 100644 index 0000000..1de73df --- /dev/null +++ b/libraries/cluster/testutils/fakehostcli.py @@ -0,0 +1,330 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +from collections import namedtuple +import six + +BASIC_MEMBER = 'basic_member' + +LINUX_USER = 'linux_user' + +ROLES = ['role{}'.format(i) for i in range(4)] + + +class UserTuple(namedtuple('UserTuple', ['uuid', + 'username', + 'email', + 'password', + 'roles'])): + def __hash__(self): + d = self._asdict().copy() + d['roles'] = sorted(d['roles']) + return hash(repr(sorted(d.items()))) + + +class User(object): + def __init__(self, username, email='user@email.me', password=None): + self.username = username + self.email = email + self._password = password + self.added_roles = set([]) + self._uuid = None + self._password_history = [self.password] + + def set_password(self, password): + self._password = password + self._password_history.append(password) + + @property + def password(self): + return self._password + + @property + def password_history(self): + return self._password_history + + @property + def uuid(self): + return self._uuid if self._uuid else 'UUID_{}'.format(self.username) + + def set_uuid(self, uuid): + self._uuid = uuid + + def set_added_roles(self, roles): + self.added_roles = set(roles) + + def add_role(self, role): + assert role not in self.roles, 'Role {role} already in {username}'.format( + role=role, + username=self.username) + self.added_roles.add(role) + + @property + def roles(self): + return self.added_roles.union(set([BASIC_MEMBER])) + + def __repr__(self): + return str(self.usertuple) # pragma: no cover + + def __eq__(self, other): + return self.usertuple == other + + def __hash__(self): + return hash(self.usertuple) + + @property + def usertuple(self): + return UserTuple(uuid=self.uuid, + username=self.username, + email=self.email, + password=self.password, + roles=self.roles) + + +class HandlerError(Exception): + pass + + +@six.add_metaclass(abc.ABCMeta) +class HandlerBase(object): + def __init__(self, users): + self._users = users + + def handle(self, cmd, target): + self._handle_target(target) + + if not cmd.startswith(self._expected_startswith): + raise HandlerError() + + return self._handle(cmd.split()) + + @abc.abstractmethod + def _handle_target(self, target): + """Handle target + + Raises: + HandlerError: if target cannot be handled by handler + """ + + @abc.abstractproperty + def _expected_startswith(self): + """Return the expected start of the command. + """ + + @abc.abstractmethod + def _handle(self, args): + """Handle and return the command + + Raises: + HandlerError: if command cannot be handled by handler + """ + + +class SetPasswordHandler(HandlerBase): + """Handler for HostCli command + user set password --opassword OPASSWORD --npassword NPASSWORD + """ + def __init__(self, users, envcreator): + super(SetPasswordHandler, self).__init__(users) + self._envcreator = envcreator + self._envname = None + + @property + def _expected_startswith(self): + return 'user set password' + + def _handle_target(self, target): + if not target.startswith('default.set_password:'): + raise HandlerError() + self._envname = target.split('.')[1] + + def _handle(self, args): + pwds = self._get_passwords(args) + openrc_dict = self._envcreator.create(target='default', envname=self._envname) + assert pwds.old == openrc_dict['OS_PASSWORD'], (pwds.old, + openrc_dict['OS_PASSWORD']) + user = self._get_user_for_openrc_dict(openrc_dict) + assert pwds.old == user.password + user.set_password(pwds.new) + return 'Your password has been changed.' + + @staticmethod + def _get_passwords(args): + assert args[3] == '--opassword' + assert args[5] == '--npassword' + p = Passwords(old=args[4], new=args[6]) + assert p.old != p.new, p + return p + + def _get_user_for_openrc_dict(self, openrc_dict): + username = openrc_dict['OS_USERNAME'] + for _, user in self._users.items(): + if user.username == username: + return user + + raise AssertionError('User {} not found'.format(username)) # pragma: no cover + + +class Passwords(namedtuple('Passwords', ['old', 'new'])): + pass + + +class AdminHandlerBase(HandlerBase): # pylint: disable=abstract-method + def _handle_target(self, target): + if target != 'default.um_admin': + raise HandlerError() + + +class UserCreateHandler(AdminHandlerBase): + @property + def _expected_startswith(self): + return 'user create' + + def _handle(self, args): + assert args[3] == '--email' + assert args[5] == '--password' + user = User(username=args[2], + email=args[4], + password=args[6]) + assert user.uuid not in self._users, 'User {} already created'.format(user) + self._users[user.uuid] = user + return 'User created. The UUID is {uuid}'.format(uuid=user.uuid) + + +class UserListHandler(AdminHandlerBase): + @property + def _expected_startswith(self): + return 'user list' + + def _handle(self, args): + return [{'Password-Expires': None, + 'User-ID': u.uuid, + 'Enabled': True, + 'User-Name': u.username} for _, u in self._users.items()] + + +class UserDeleteHandler(AdminHandlerBase): + @property + def _expected_startswith(self): + return 'user delete' + + def _handle(self, args): + assert len(args) == 3, 'Wrong number of arguments for delete {}'.format(args) + del self._users[args[2]] + return 'User deleted.' + + +class CorruptedUserListHandler(UserListHandler): + def _handle(self, args): + return [] + + +class UserAddRoleHandler(AdminHandlerBase): + @property + def _expected_startswith(self): + return 'user add role' + + def _handle(self, args): + self._users[args[3]].add_role(args[4]) + return 'Role has been added to the user.' + + +class RoleListAllHandler(AdminHandlerBase): + def __init__(self, users, role_attr): + super(RoleListAllHandler, self).__init__(users) + self._role_attr = role_attr + + @property + def _expected_startswith(self): + return 'role list all' + + def _handle(self, args): + assert len(args) == 3, 'Expected: {expected}, actual {actual}'.format( + expected=self._expected_startswith.split(), + actual=args) + return [{'Role-Description': 'Role Description {}'.format(role), + 'Chroot': False, + 'Is-Service-Role': True, + self._role_attr: role} for role in self._roles] + + @property + def _roles(self): + return ROLES + [BASIC_MEMBER, LINUX_USER] + + +class Handlers(object): # pylint: disable=too-few-public-methods + def __init__(self, handlers): + self._handlers = handlers + + def handle(self, cmd, target): + for h in self._handlers: + try: + return h.handle(cmd=cmd, target=target) + except HandlerError: + pass + + assert 0, 'User Command {!r} not found'.format(cmd) # pragma: no cover + + +class FakeHostCliUser(object): + def __init__(self, mock_hostcli, role_attr): + self._mock_hostcli = mock_hostcli + self._role_attr = role_attr + self._envcreator = None + self._users = {} + self._run_raw_handlers = None + self._run_handlers = None + + def set_envcreator(self, envcreator): + self._envcreator = envcreator + + def initialize(self): + self._run_raw_handlers = self._create_handlers(UserCreateHandler, + UserAddRoleHandler, + UserDeleteHandler, + self._create_set_password_handler) + self._run_handlers = self._create_handlers(UserListHandler, + self._create_role_handler) + self._set_side_effects() + + @property + def users(self): + return set([u for _, u in self._users.items()]) + + def get_user(self, user_id): + return self._users[user_id] + + def set_corrupted_user_list(self): + self._run_handlers = self._create_handlers(CorruptedUserListHandler, + self._create_role_handler) + + def _create_role_handler(self, users): + return RoleListAllHandler(users, role_attr=self._role_attr) + + def _create_set_password_handler(self, users): + return SetPasswordHandler(users, envcreator=self._envcreator) + + def _create_handlers(self, *handler_factories): + return Handlers([h(self._users) for h in handler_factories]) + + def _set_side_effects(self): + self._mock_hostcli.return_value.run.side_effect = self._run + self._mock_hostcli.return_value.run_raw.side_effect = self._run_raw + + def _run_raw(self, cmd, target): + return self._run_raw_handlers.handle(cmd, target=target) + + def _run(self, cmd, target): + return self._run_handlers.handle(cmd, target=target) diff --git a/libraries/cluster/testutils/host.py b/libraries/cluster/testutils/host.py new file mode 100644 index 0000000..d9d97ac --- /dev/null +++ b/libraries/cluster/testutils/host.py @@ -0,0 +1,162 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import six +from . profiles import ( + MasterProfile, + WorkerProfile, + StorageProfile, + BaseProfile, + ManagementProfile) + + +class Host(object): + def __init__(self, name, service_profiles, ippool): + self.name = name + self.service_profiles = service_profiles + self.internal_ip = ippool.get_internal_ip() + self.external_ip = ippool.get_external_ip() + + @property + def host_dict(self): + d = self._service_profiles_dict + d.update({'network_domain': self.expected_network_domain}) + d.update(self._network_profiles_dict) + return d + + @property + def _service_profiles_dict(self): + return {'service_profiles': [str(s()) for s in self.service_profiles]} + + @property + def expected_network_domain(self): + return '{name}-domain-name'.format(name=self.name) + + @property + def _network_profiles_dict(self): + return {'network_profiles': self._network_profiles} + + @property + def _network_profiles(self): + return ['network_nondpdk'] + + @property + def networking(self): + return {'infra_internal': {'ip': self.internal_ip}, + 'infra_external': {'ip': self.external_ip}} + + @property + def network_profile_details(self): + return { + 'network_nondpdk': { + 'provider_network_interfaces': { + 'bond1': { + 'type': 'ovs'}}}, + 'network_empty': {}} + + @staticmethod + def expected_is_dpdk(): + return False + + +class DpdkHost(Host): + + @property + def _network_profiles(self): + return ['network_dpdk'] + + @property + def network_profile_details(self): + return { + 'network_dpdk': { + 'provider_network_interfaces': { + 'bond1': { + 'type': 'ovs-dpdk'}}}} + + @staticmethod + def expected_is_dpdk(): + return True + + +@six.add_metaclass(abc.ABCMeta) +class HostGeneratorBase(object): + """ + Abstract generator base for :class:`.Host` instances. + + Arguments: + ippool: :class:`IPPool` instance + """ + def __init__(self, ippool): + self._ippool = ippool + + @property + def _host_cls(self): + return Host + + def gen(self, nmbr, service_profiles=None, start=1): + service_profiles = service_profiles or self._default_profiles + for i in range(start, start + nmbr): + yield self._host_cls( + name='{basename}-{i}'.format(basename=self._basename, i=i), + service_profiles=service_profiles, + ippool=self._ippool) + + @abc.abstractproperty + def _basename(self): + """Return basename of the host e.g. master""" + + @abc.abstractproperty + def _default_profiles(self): + """Return iterable of service profiles""" + + +class MasterGenerator(HostGeneratorBase): + @property + def _basename(self): + return 'master' + + @property + def _default_profiles(self): + return [MasterProfile, + WorkerProfile, + StorageProfile, + BaseProfile, + ManagementProfile] + + +class WorkerGenerator(HostGeneratorBase): + @property + def _basename(self): + return 'worker' + + @property + def _default_profiles(self): + return [WorkerProfile, BaseProfile] + + +class StorageGenerator(HostGeneratorBase): + @property + def _basename(self): + return 'storage' + + @property + def _default_profiles(self): + return [StorageProfile] + + +class DpdkWorkerGenerator(WorkerGenerator): + @property + def _host_cls(self): + return DpdkHost diff --git a/libraries/cluster/testutils/ippool.py b/libraries/cluster/testutils/ippool.py new file mode 100644 index 0000000..1f42100 --- /dev/null +++ b/libraries/cluster/testutils/ippool.py @@ -0,0 +1,32 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + + +class IPPool(object): + def __init__(self): + self._internal_ip = self._ip_gen(base='192.168.1') + self._external_ip = self._ip_gen(base='10.1.1') + + def get_internal_ip(self): + return next(self._internal_ip) + + def get_external_ip(self): + return next(self._external_ip) + + @staticmethod + def _ip_gen(base): + for i in itertools.count(start=2): # pragma: no branch + yield '{base}.{i}'.format(base=base, i=i) diff --git a/libraries/cluster/testutils/profiles.py b/libraries/cluster/testutils/profiles.py new file mode 100644 index 0000000..99bea38 --- /dev/null +++ b/libraries/cluster/testutils/profiles.py @@ -0,0 +1,38 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class MasterProfile(object): + def __str__(self): + return 'test_master' + + +class WorkerProfile(object): + def __str__(self): + return 'test_worker' + + +class StorageProfile(object): + def __str__(self): + return 'storage' + + +class BaseProfile(object): + def __str__(self): + return 'base' + + +class ManagementProfile(object): + def __str__(self): + return 'management' diff --git a/libraries/cluster/testutils/user_config.yaml b/libraries/cluster/testutils/user_config.yaml new file mode 100644 index 0000000..5e1542a --- /dev/null +++ b/libraries/cluster/testutils/user_config.yaml @@ -0,0 +1,16 @@ +version: 2.0.0 +name: cluster-name + +### Cloud description +description: Cloud description + +time: + ntp_servers: [ 10.1.1.2, 10.1.1.3 ] + zone: Europe/Helsinki + +users: + admin_user_name: cloudadmin + admin_user_password: "crypted password" + initial_user_name: um_admin_username + initial_user_password: um_admin_password + admin_password: admin_password diff --git a/libraries/cluster/usergen.py b/libraries/cluster/usergen.py new file mode 100644 index 0000000..5bcdb5d --- /dev/null +++ b/libraries/cluster/usergen.py @@ -0,0 +1,39 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + + +class UserGen(object): + + def __init__(self, roles_len): + self._roles_len = roles_len + self._usergens = {} + + def create_username(self, roles): + if not roles: + return 'no_roles' + if len(roles) == self._roles_len: + return 'all_roles' + return next(self._get_user_gen(sorted(roles)[0])) + + def _get_user_gen(self, base): + if base not in self._usergens: + self._usergens[base] = self._user_gen(base) + return self._usergens[base] + + @staticmethod + def _user_gen(base): + for idx in itertools.count(): # pragma: no branch + yield '{base}{idx}'.format(base=base, idx=idx) diff --git a/libraries/cluster/usermanager.py b/libraries/cluster/usermanager.py new file mode 100644 index 0000000..903d02b --- /dev/null +++ b/libraries/cluster/usermanager.py @@ -0,0 +1,178 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +from .usergen import UserGen + +INITIAL_PASSWORD = 'intl_AM10srt' +PASSWORD = 'um_UM0scrt' + + +class UserRecord(namedtuple('UserRecord', ['uuid', + 'username', + 'password', + 'roles'])): + + def create_with_password(self, password): + d = self._asdict().copy() + d.update({'password': password}) + return UserRecord(**d) + + +class UserManagerError(Exception): + pass + + +class UserManager(object): + + _target = 'default.um_admin' + _all_roles = 'all_roles' + _no_roles = 'no_roles' + _special_roles = [_all_roles, _no_roles] + _basic_member = 'basic_member' + _linux_user = 'linux_user' + + def __init__(self, hostcli_factory): + self._hostcli_factory = hostcli_factory + self._hostcli_inst = None + self._roles_users = dict() + self._roles_cache = set() + self._user_gen_inst = None + + def create_user_with_roles(self, *roles): + r = self._get_and_verify_roles(roles) + + key = self._get_roles_users_key(r) + if key not in self._roles_users: + self._roles_users[key] = self._create_with_roles(r) + self._change_password(roles) + + return self._roles_users[key] + + def delete_users(self): + """Delete all users created by :meth:`.create_user_with_roles`. + """ + for _, userrecord in self._roles_users.items(): + self._hostcli.run_raw('user delete {}'.format(userrecord.uuid), + target=self._target) + + def get_user_and_set_password(self, *roles): + """Get user with INITIAL_PASSWORD and set password to PASSWORD. + + .. note:: + + This method should be called only by :class:`cluster.envcreator.EnvCreator`. + """ + upd_roles = self._get_roles(roles) + key = self._get_roles_users_key(upd_roles) + old_userrecord = self._roles_users[key] + self._roles_users[key] = old_userrecord.create_with_password(PASSWORD) + return old_userrecord + + @staticmethod + def _get_roles_users_key(roles): + return frozenset(roles) + + @property + def _hostcli(self): + if self._hostcli_inst is None: + self._hostcli_inst = self._hostcli_factory() + return self._hostcli_inst + + def _get_uuid(self, username): + users = self._hostcli.run('user list', target=self._target) + for u in users: + if u['User-Name'] == username: + return u['User-ID'] + + raise UserManagerError('User {} does not exist in target'.format(username)) + + def _get_and_verify_roles(self, roles): + self._verify_special_roles(roles) + r = self._get_roles(roles) + self._verify_roles(r) + return r + + def _verify_special_roles(self, roles): + if len(roles) == 1: + return + + for special_role in self._special_roles: + if special_role in roles: + raise UserManagerError( + 'Special role {special_role!r} and other roles in {roles}'.format( + special_role=special_role, + roles=roles)) + + def _get_roles(self, roles): + if set(roles) == set([self._all_roles]): + return self._roles + if set(roles) == set([self._no_roles]): + return [] + return roles + + def _verify_roles(self, roles): + given_roles = set(roles) + if len(roles) > len(given_roles): + raise UserManagerError('Duplicate roles in {}'.format(roles)) + target_roles = set(self._roles) + notexisting = given_roles - target_roles + if notexisting: + raise UserManagerError('Roles {} not found'.format(notexisting)) + + def _create_with_roles(self, roles): + username = self._user_gen.create_username(roles) + uuid = self._create_user_from_user(username, roles) + return UserRecord(uuid=uuid, + username=username, + password=INITIAL_PASSWORD, + roles=roles) + + def _create_user_from_user(self, username, roles): + self._hostcli.run_raw('user create {username} ' + '--email user@email.me ' + '--password {password}'.format( + username=username, + password=INITIAL_PASSWORD), target=self._target) + uuid = self._get_uuid(username) + for role in roles: + self._hostcli.run_raw('user add role {uuid} {role}'.format(uuid=uuid, + role=role), + target=self._target) + return uuid + + def _change_password(self, roles): + self._hostcli.run_raw( + 'user set password --opassword {old} --npassword {new}'.format( + old=INITIAL_PASSWORD, + new=PASSWORD), + target='default.set_password:{}'.format(','.join(roles))) + + @property + def _user_gen(self): + if self._user_gen_inst is None: + self._user_gen_inst = UserGen(len(self._roles)) + return self._user_gen_inst + + @property + def _roles(self): + if not self._roles_cache: + self._roles_cache = self._get_roles_via_hostcli() + return self._roles_cache + + def _get_roles_via_hostcli(self): + roles = self._hostcli.run('role list all', target=self._target) + role_attr = 'Role-Name' if 'Role-Name' in roles[0] else 'Role' + return [role[role_attr] for role in roles + if role[role_attr] not in [self._basic_member, self._linux_user]] diff --git a/libraries/cluster/userverifier.py b/libraries/cluster/userverifier.py new file mode 100644 index 0000000..a2d2f73 --- /dev/null +++ b/libraries/cluster/userverifier.py @@ -0,0 +1,160 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import mock +import pytest +from crl.remotesession.remotesession import RemoteSession +from hostcli import HostCli +from .usermanager import ( + UserManager, + UserManagerError, + PASSWORD, + INITIAL_PASSWORD) +from .envcreator import EnvCreator +from .testutils.fakehostcli import ( + FakeHostCliUser, + User, + ROLES) +from .usergen import UserGen + + +class UserVerifier(object): + def __init__(self, role_attr): + self._mock_hostcli_factory = mock.create_autospec(HostCli) + self._mock_remotesession_factory = mock.create_autospec(RemoteSession) + self._hostcliuser = FakeHostCliUser(self._mock_hostcli_factory, + role_attr=role_attr) + self._usermanager = UserManager(self._mock_hostcli_factory) + self._envcreator = EnvCreator( + remotesession_factory=self._mock_remotesession_factory, + usermanager=self._usermanager) + self._usergen = None + self._initialize() + + def _initialize(self): + self._setup_remotesession() + self._setup_hostcliuser() + self._setup_usergen() + + def _setup_remotesession(self): + g = self._mock_remotesession_factory.return_value.get_source_update_env_dict + g.return_value = {} + + def _setup_hostcliuser(self): + self._hostcliuser.set_envcreator(self._envcreator) + self._hostcliuser.initialize() + + def _setup_usergen(self): + self._usergen = UserGen(len(ROLES)) + + def verify_create_user_with_roles(self): + for roles in self._roles_gen(): + actual_user = self._get_actual_user( + self._usermanager.create_user_with_roles(*roles)) + expected_user = self._get_expected_user(roles) + for actual in [actual_user, self._hostcliuser.get_user(expected_user.uuid)]: + assert actual_user == expected_user, ( + 'expected {expected}, actual {actual}'.format( + expected=expected_user, + actual=actual)) + self._assert_password_history(actual_user.uuid) + + def verify_delete_users(self): + self.verify_create_user_with_roles() + self._usermanager.delete_users() + assert not self._hostcliuser.users + + def verify_corrupted_user_list(self): + self._hostcliuser.set_corrupted_user_list() + with pytest.raises(UserManagerError) as excinfo: + self._usermanager.create_user_with_roles(*ROLES) + + assert str(excinfo.value) == 'User all_roles does not exist in target' + + def verify_user_with_roles_notexist(self): + notexists = ['notexists'] + with pytest.raises(UserManagerError) as excinfo: + self._usermanager.create_user_with_roles(ROLES[0], *notexists) + + msg = str(excinfo.value) + assert msg == 'Roles {} not found'.format(set(notexists)), msg + + def verify_user_roles_duplicates(self): + duplicates = (ROLES[0], ROLES[1], ROLES[0]) + with pytest.raises(UserManagerError) as excinfo: + self._usermanager.create_user_with_roles(*duplicates) + + msg = str(excinfo.value) + assert msg == 'Duplicate roles in {}'.format(duplicates), msg + + def verify_one_user_per_roles(self): + users_list = [] + for _ in range(2): + self._setup_usergen() + self.verify_create_user_with_roles() + users_list.append(self._hostcliuser.users) + + assert users_list[0] == users_list[1], users_list + + def verify_all_roles(self): + userrecord = self._usermanager.create_user_with_roles('all_roles') + user = self._hostcliuser.get_user(userrecord.uuid) + assert user.username == 'all_roles', user.username + assert user.added_roles == set(ROLES) + + def verify_no_roles(self): + userrecord = self._usermanager.create_user_with_roles('no_roles') + user = self._hostcliuser.get_user(userrecord.uuid) + assert user.username == 'no_roles', user.username + assert not user.added_roles + + def verify_special_roles_raises(self): + for special_role in ['no_roles', 'all_roles']: + with pytest.raises(UserManagerError) as excinfo: + roles = (special_role, ROLES[0]) + self._usermanager.create_user_with_roles(*roles) + + msg = str(excinfo.value) + assert msg == 'Special role {special_role!r} and other roles in {roles}'.format( + special_role=special_role, + roles=roles), msg + + @staticmethod + def _get_actual_user(user): + u = User(username=user.username, password=user.password) + u.set_added_roles(user.roles) + u.set_uuid(user.uuid) + return u + + def _get_expected_user(self, roles): + user = User(username=self._usergen.create_username(roles), + password=PASSWORD) + user.set_added_roles(roles) + return user + + @staticmethod + def _roles_gen(): + for r in range(len(ROLES) + 1): + for roles in itertools.combinations(ROLES, r): + yield roles + + def _assert_password_history(self, user_id): + user = self._hostcliuser.get_user(user_id) + actual_history = user.password_history + expected_history = [INITIAL_PASSWORD, PASSWORD] + assert actual_history == expected_history, ( + 'Expected {expected}, actual {actual}'.format( + expected=expected_history, + actual=actual_history)) diff --git a/libraries/hostcli/__init__.py b/libraries/hostcli/__init__.py new file mode 100644 index 0000000..69fca60 --- /dev/null +++ b/libraries/hostcli/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .hostcli import HostCli +from .hostcliuser import HostCliUser diff --git a/libraries/hostcli/hostcli.py b/libraries/hostcli/hostcli.py new file mode 100644 index 0000000..282f862 --- /dev/null +++ b/libraries/hostcli/hostcli.py @@ -0,0 +1,23 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from openstackcli import OpenStack + + +class HostCli(OpenStack): + """ + The same as OpenStack_ but the remote command executed is *hostcli*. + + .. _OpenStack: openstackcli.OpenStack.html + """ diff --git a/libraries/hostcli/hostcliuser.py b/libraries/hostcli/hostcliuser.py new file mode 100644 index 0000000..f4380b0 --- /dev/null +++ b/libraries/hostcli/hostcliuser.py @@ -0,0 +1,64 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from openstackcli import ( + OpenStack, + Runner) +from .hostcli import HostCli + + +class HostCliUser(object): # pylint: disable=too-few-public-methods + """Base class for users of the + - :class:`OpenStack` + - :class:`Runner` + - `crl.remotesession.remotesession.RemoteSession` + + instances. + + Attributes: + + _hostcli: :class:`HostCli` instance + + _openstack: :class:`.OpenStack` instance + + _runner: :class:`.Runner` instance + + _remotesession: :class:`crl.remotesession.remotesession.RemoteSession` + instance. + + """ + def __init__(self): + self._hostcli = HostCli() + self._openstack = OpenStack() + self._runner = Runner() + self._remotesession = None + self._envname = None + + def initialize(self, remotesession, envname=None): + self._remotesession = remotesession + self._envname = envname + envkwargs = {} if envname is None else {'envname': envname} + for runner in [self._hostcli, self._openstack, self._runner]: + runner.initialize(remotesession, **envkwargs) + + def _get_env_target(self, target='default'): + """Get ennvironment target for the *target*. + """ + return '{target}{env_postfix}'.format( + target=target, + env_postfix=self._env_postfix) + + @property + def _env_postfix(self): + return '.{}'.format(self._envname) if self._envname else '' diff --git a/libraries/hostcli/test_hostcli.py b/libraries/hostcli/test_hostcli.py new file mode 100644 index 0000000..4b27f60 --- /dev/null +++ b/libraries/hostcli/test_hostcli.py @@ -0,0 +1,37 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name +import pytest +import mock +from crl.remotesession.remotesession import RemoteSession +from crl.interactivesessions.remoterunner import RunResult +from .hostcli import HostCli + + +@pytest.fixture +def mock_remotesession(): + m = mock.create_autospec(RemoteSession) + m.execute_command_in_target.return_value = RunResult(status=0, + stdout='"result"', + stderr='') + return m + + +def test_hostcli(mock_remotesession): + h = HostCli() + h.initialize(mock_remotesession) + assert h.run('cmd') == 'result' + mock_remotesession.execute_command_in_target.assert_called_once_with( + 'hostcli --os-cloud default cmd -f json', target='default') diff --git a/libraries/hostcli/test_hostcliuser.py b/libraries/hostcli/test_hostcliuser.py new file mode 100644 index 0000000..2596640 --- /dev/null +++ b/libraries/hostcli/test_hostcliuser.py @@ -0,0 +1,128 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name +from collections import namedtuple +import mock +import pytest +from openstackcli import ( + OpenStack, + Runner) +from . import hostcliuser +from .hostcli import HostCli +from .hostcliuser import HostCliUser + + +@pytest.fixture +def mock_hostcli(): + with mock.patch.object(hostcliuser, 'HostCli', + mock.create_autospec(HostCli)) as p: + yield p + + +@pytest.fixture +def mock_openstack(): + with mock.patch.object(hostcliuser, + 'OpenStack', + mock.create_autospec(OpenStack)) as p: + yield p + + +@pytest.fixture +def mock_runner(): + with mock.patch.object(hostcliuser, + 'Runner', + mock.create_autospec(Runner)) as p: + yield p + + +@pytest.fixture(params=[{}, + {'envname': 'envname'}, + {'envname': 'otherenv'}]) +def envkwargs(request): + return request.param + + +@pytest.fixture +def examplehostcliuser(mock_remotesession, + mock_runners, + envkwargs): + e = ExampleHostCliUser(mock_runners, + mock_remotesession=mock_remotesession, + envkwargs=envkwargs) + e.initialize(mock_remotesession, **envkwargs) + for runner in mock_runners: + initialize = runner.return_value.initialize + initialize.assert_called_once_with(mock_remotesession, **envkwargs) + return e + + +@pytest.fixture +def mock_runners(mock_hostcli, mock_openstack, mock_runner): + return MockRunners(hostcli=mock_hostcli, + openstack=mock_openstack, + runner=mock_runner) + + +class ExampleHostCliUser(HostCliUser): + + def __init__(self, mock_runners, mock_remotesession, envkwargs): + super(ExampleHostCliUser, self).__init__() + self._mock_runners = mock_runners + self._mock_remotesession = mock_remotesession + self._envkwargs = envkwargs + + @property + def cmd(self): + return 'cmd' + + def verify_runs(self): + target = 'target' + for rm in self._runnermocks: + mock_run = rm.mock.return_value.run + assert rm.runner.run(self.cmd, target=target) == mock_run.return_value + mock_run.assert_called_once_with(self.cmd, target=target) + + execute = self._mock_remotesession.execute_command_in_target + assert self._remotesession.execute_command_in_target( + self.cmd, target=target) == execute.return_value + + @property + def _runnermocks(self): + yield RunnerMock(runner=self._hostcli, mock=self._mock_runners.hostcli) + yield RunnerMock(runner=self._openstack, mock=self._mock_runners.openstack) + yield RunnerMock(runner=self._runner, mock=self._mock_runners.runner) + + def verify_get_env_target(self): + postfix = ('.{}'.format(self._envkwargs['envname']) + if self._envkwargs else + '') + assert self._get_env_target() == 'default{}'.format(postfix) + assert self._get_env_target('target') == 'target{}'.format(postfix) + + +class RunnerMock(namedtuple('RunMock', ['runner', 'mock'])): + pass + + +class MockRunners(namedtuple('MockRunners', ['hostcli', 'openstack', 'runner'])): + pass + + +def test_hostcliuser_runners(examplehostcliuser): + examplehostcliuser.verify_runs() + + +def test_get_env_target(examplehostcliuser): + examplehostcliuser.verify_get_env_target() diff --git a/libraries/openstackcli/__init__.py b/libraries/openstackcli/__init__.py new file mode 100644 index 0000000..bef34d7 --- /dev/null +++ b/libraries/openstackcli/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . openstackcli import ( + OpenStack, + Runner, + OpenStackCliError) diff --git a/libraries/openstackcli/cliwrapperbase.py b/libraries/openstackcli/cliwrapperbase.py new file mode 100644 index 0000000..36e046f --- /dev/null +++ b/libraries/openstackcli/cliwrapperbase.py @@ -0,0 +1,101 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class CliWrapperBase(object): + """Test class base for CLI (openstackcli). + """ + def __init__(self, clicls): + self._cli = clicls() + self._remotesession = None + self._exec_func = None + self._expected_cmd_postfix = '' + + @property + def cli(self): + return self._cli + + def set_remotesession(self, remotesession): + """Set *RemoteSession* mock class instance. + """ + self._remotesession = remotesession + + @property + def remotesession(self): + return self._remotesession + + def set_expected_cmd_postfix(self, expected_cmd_postfix): + """Set expected cmd postfix for *RemoteSession* *exec_func* call. + """ + self._expected_cmd_postfix = expected_cmd_postfix + + def set_exec_func_and_return_value(self, exec_func, return_value): + """Set expected *RunnerSession* execution function and + set mock return value for this call. + """ + self._exec_func = exec_func + self._exec_func.return_value = return_value + + def run_with_verify(self, run_method, cmd): + """Run *run_method* of CLI with *cmd* and *_target_kwargs* kwargs. Then + verify the *RemoteSession* *_exec_func* call. + + Return: + *run_method* return value. + """ + ret = run_method(cmd, **self._target_kwargs) + self._exec_func.assert_called_once_with( + self._get_expected_cmd(cmd + self._expected_cmd_postfix), + target=self._expected_target) + return ret + + @abc.abstractproperty + def _expected_target(self): + """Return expected target for RemoteSession call. + """ + + def _get_expected_cmd(self, cmd): + return '{pre_cmd}{clistr}{expected_cmd_args}{cmd}'.format( + pre_cmd=self._pre_cmd, + clistr=self._clistr, + expected_cmd_args=self._expected_cmd_args, + cmd=cmd) + + @property + def _clistr(self): + n = self._cli.__class__.__name__ + return '' if n == 'Runner' else n.lower() + + @property + def _pre_cmd(self): + return '' + + @abc.abstractproperty + def _target_kwargs(self): + """Return target kwargs for *RemoteSession* method call. + """ + + @abc.abstractproperty + def _expected_cmd_args(self): + """Return args string after cli. + """ + + def initialize(self): + """Initialize CLI with mock RemoteSession instance *remotesession*. + """ + self._cli.initialize(self.remotesession) diff --git a/libraries/openstackcli/cloudcli.py b/libraries/openstackcli/cloudcli.py new file mode 100644 index 0000000..b1ebc0d --- /dev/null +++ b/libraries/openstackcli/cloudcli.py @@ -0,0 +1,81 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import six +from .cliwrapperbase import CliWrapperBase + + +@six.add_metaclass(abc.ABCMeta) # pylint: disable=abstract-method +class CloudCliBase(CliWrapperBase): + + @property + def _target_kwargs(self): + return {'target': '{prebase}{os_cloud}.{cloudname}'.format( + prebase=self._prebase, + os_cloud=self._os_cloud, + cloudname=self._cloudname)} + + @property + def _os_cloud(self): + return 'os-cloud' + + @property + def _prebase(self): + return ('' + if self._expected_target == 'default' else + '{}.'.format(self._expected_target)) + + @property + def _cloudname(self): + return 'cloudname' + + @property + def _expected_cmd_args(self): + return ' --os-cloud {} '.format(self._cloudname) + + +class TargetCloud(CloudCliBase): + + @property + def _expected_target(self): + return 'target' + + +class DefaultCloud(CloudCliBase): + + @property + def _expected_target(self): + return 'default' + + +class TypoCloud(CloudCliBase): + + @property + def _expected_target(self): + return 'target.{os_cloud}.{cloudname}'.format( + os_cloud=self._os_cloud, + cloudname=self._cloudname) + + @property + def _prebase(self): + return 'target.' + + @property + def _os_cloud(self): + return 'typo-cloud' + + @property + def _expected_cmd_args(self): + return ' --os-cloud default ' diff --git a/libraries/openstackcli/conftest.py b/libraries/openstackcli/conftest.py new file mode 100644 index 0000000..ec4214b --- /dev/null +++ b/libraries/openstackcli/conftest.py @@ -0,0 +1,67 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name +from collections import namedtuple + +import pytest +from .openstackcli import ( + OpenStack, + Runner) + + +from .cloudcli import ( + DefaultCloud, + TargetCloud, + TypoCloud) + +from .envcli import ( + EnvCli, + InitializedEnvCli) + + +CLIWRAPPER_CLSES = [DefaultCloud, + TargetCloud, + TypoCloud, + EnvCli, + InitializedEnvCli] + + +CLI_CLSES = [OpenStack, Runner] + + +class WrapperCliTuple(namedtuple('WrapperCliTuple', ['wrapper', 'cli'])): + pass + + +@pytest.fixture(params=[WrapperCliTuple(wrapper=cliw, cli=clicls) + for clicls in CLI_CLSES + for cliw in CLIWRAPPER_CLSES]) +def cliwrapper(mock_remotesession, request): + c = request.param.wrapper(request.param.cli) + c.set_remotesession(mock_remotesession) + c.initialize() + return c + + +class MethodFmt(namedtuple('MethodFormat', ['method', 'fmt'])): + pass + + +@pytest.fixture(params=['run', 'run_ignore_output']) +def methodfmt(request, openstack): + return { + 'run': MethodFmt(openstack.run, ' -f json'), + 'run_ignore_output': MethodFmt(openstack.run_ignore_output, '')}[ + request.param] diff --git a/libraries/openstackcli/envcli.py b/libraries/openstackcli/envcli.py new file mode 100644 index 0000000..cd6c985 --- /dev/null +++ b/libraries/openstackcli/envcli.py @@ -0,0 +1,40 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cliwrapperbase import CliWrapperBase + + +class EnvCli(CliWrapperBase): + + @property + def _target_kwargs(self): + return {'target': self._expected_target} + + @property + def _expected_target(self): + return 'target.env' + + @property + def _expected_cmd_args(self): + return ' ' + + +class InitializedEnvCli(EnvCli): + + @property + def _target_kwargs(self): + return {'target': 'target'} + + def initialize(self): + self._cli.initialize(self.remotesession, envname='env') diff --git a/libraries/openstackcli/openstackcli.py b/libraries/openstackcli/openstackcli.py new file mode 100644 index 0000000..72559ca --- /dev/null +++ b/libraries/openstackcli/openstackcli.py @@ -0,0 +1,404 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test library for running *openstack* in remote system. +""" +import abc +import json +from collections import namedtuple +from contextlib import contextmanager +import six + + +class OpenStackCliError(Exception): + """Exception raised in case CLI in the remote system fails. + """ + pass + + +class _FailedMessage(namedtuple('FailedMessage', ['cmd', 'target'])): + def __str__(self): + return "Remote command '{cmd}' in target '{target}' failed".format( + cmd=self.cmd, + target=self.target) + + +@six.add_metaclass(abc.ABCMeta) +class _TargetBase(object): + + @abc.abstractproperty + def target(self): + """Return *RemoteSession* target where command is to be executed + """ + + @abc.abstractproperty + def cmd_template(self): + """Return cmd template with field *cli*, *cmd* and *fmt*. + """ + + +class _DefaultTarget(_TargetBase): + def __init__(self): + self._raw_target = None + self._target = None + self._cloud = 'default' + + def set_target(self, target): + self._raw_target = target + self._setup() + + def _setup(self): + self._target = self._raw_target + if self._partslen == 2 or self._partslen == 3: + self._update_target_and_cloud() + + def _update_target_and_cloud(self): + if self._oper == 'os-cloud': + self._target = self._parts[0] if self._partslen == 3 else 'default' + self._cloud = self._parts[-1] + elif self._partslen == 2: + self._cloud = None + + @property + def _oper(self): + return self._parts[self._partslen - 2] + + @property + def _parts(self): + return self._raw_target.split('.') + + @property + def _partslen(self): + return len(self._parts) + + @property + def target(self): + return self._target + + @property + def cmd_template(self): + return ('{cli} {cmd}{fmt}' + if self._cloud is None else + '{{cli}} --os-cloud {cloud} {{cmd}}{{fmt}}'.format(cloud=self._cloud)) + + +class _CliRunnerInTarget(object): + def __init__(self, run, target): + self._run = run + self._target = target + self._cli = None + + def set_cli(self, cli): + self._cli = cli + + def run_with_json(self, cmd): + return self._run_with_verification(cmd, fmt=' -f json') + + def run(self, cmd): + return self._run_with_verification(cmd) + + def _run_with_verification(self, cmd, fmt=''): + cmd = self._get_formatted_cmd(cmd, fmt) + result = self._run(cmd, self._target.target) + self._verify_result(result, _FailedMessage(cmd, self._target.target)) + return result + + def run_without_verification(self, cmd, fmt=''): + return self._run(self._get_formatted_cmd(cmd, fmt), self._target.target) + + def _get_formatted_cmd(self, cmd, fmt): + return self._target.cmd_template.format(cli=self._cli, + cmd=cmd, + fmt=fmt) + + def _verify_result(self, result, failedmessage): + status = self._get_integer_status(result, failedmessage) + if status or result.stderr: + self._raise_openstackerror(failedmessage, result) + return result + + def _get_integer_status(self, result, failedmessage): + try: + return int(result.status) + except ValueError: + self._raise_openstackerror(failedmessage, result) + + @staticmethod + def _raise_openstackerror(failedmessage, result): + raise OpenStackCliError( + '{failedmessage}: status: {status!r}, ' + 'stdout: {stdout!r}, ' + 'stderr: {stderr!r}'.format(failedmessage=failedmessage, + status=result.status, + stdout=result.stdout, + stderr=result.stderr)) + + +class OpenStack(object): + """Remote *openstack* runner. + The runner executes commands in *crl.remotesession* target but the target + *os-cloud.cloudconfigname* is interpreted to *--os-cloud cloudconfigname* + argument and executed in the *default* target. Respectively, the target + *controller-1.os-cloud.cloudconfigname* is executed in *controller-1* + target with *--os-cloud cloudconfigname*. + + If the *target* is of form *target.envname*, then *--os-cloud* is not given + as *crl.remotesession* environment handling should then take care of + setting the correct environment for the *envname* in *target*. + """ + + def __init__(self): + self._remotesession = None + self._envname = None + self._openrc_tuples = {} + + def initialize(self, remotesession, envname=None): + """Initialize the library. + + Args: + remotesession: `crl.remotesession.remotesession.RemoteSession`_ + instance + + envname: Environment name appended to the target in form target.envname. + See Robot example 2 for details of the usage. + + **Note:** + + Targets are interpreted in the following manner: + + ============================== ==================== ==================== + Target RemoteSession target os-cloud config name + ============================== ==================== ==================== + os-cloud.confname default confname + controller-1.os-cloud.confname controller-1 confname + target.envname target.envname None + ============================== ==================== ==================== + + **Robot Examples** + + In the following example is assumed that + `crl.remotesession.remotesession.RemoteSession`_ is imported in + Library settings *WITH NAME* RemoteSession. + + + *Example 1* + + ==================== ===================== ============= + ${remotesession}= Get Library Instance RemoteSession + OpenStack.Initialize ${remotesession} + ==================== ===================== ============= + + *Example 2* + + If in the test setup initialization is done with given *myenv* environment + + ==================== ===================== ============= + ${remotesession}= Get Library Instance RemoteSession + OpenStack.Initialize ${remotesession} envname=myenv + ==================== ===================== ============= + + Then + + ============= ========== ============= + OpenStack.Run quota show target=target + ============= ========== ============= + + runs *openstack quota show* in the target *target.myenv*. + + .. _crl.remotesession.remotesession.RemoteSession: + https://crl-remotesession.readthedocs.io/en/latest + /crl.remotesession.remotesession.RemoteSession.html + """ + self._remotesession = remotesession + self._envname = envname + + def run(self, cmd, target='default'): + """ Run *openstack* in the remote target with *json* format. + + Args: + cmd: openstack command to be executed in target. + + target: `crl.remotesession.remotesession.RemoteSession`_ target. + + Returns: + Decoded *json* formatted command output. For example in case + openstack output ( + for command *openstack.run('quota show')* is in remote:: + + # openstack --os-cloud default quota show -f json + { + "secgroups": 10, + "health_monitors": null, + "l7_policies": null, + ... + } + + then the return value of *run* is: + + .. code-block:: python + + { + "secgroups": 10, + "health_monitors": None, + "17_policies": None, + ... + } + + **Robot Example:** + + ================ ===================== =========== + ${quota}= OpenStack.Run quota show + Should Be Equal ${quota['secgroups']} 10 + ================ ===================== =========== + + Raises: + OpenStackCliError: if remote *openstack* fails. + + .. _crl.remotesession.remotesession.RemoteSession: + https://crl-remotesession.readthedocs.io/en/latest + /crl.remotesession.remotesession.RemoteSession.html + """ + result = self._create_runner(target).run_with_json(cmd) + with self._error_handling(cmd, target): + return json.loads(result.stdout) + + def run_ignore_output(self, cmd, target='default'): + """ Run *openstack* in the remote target and ignore the output. + + Args: + cmd: openstack command to be executed in target. + + target: `crl.remotesession.remotesession.RemoteSession`_ target. + + Returns: + Nothing + + Raises: + OpenStackCliError: if remote *openstack* fails. + + .. _crl.remotesession.remotesession.RemoteSession: + https://crl-remotesession.readthedocs.io/en/latest + /crl.remotesession.remotesession.RemoteSession.html + """ + self._create_runner(target).run(cmd) + + def run_raw(self, cmd, target='default'): + """ Run *openstack* in the remote target and return raw output without + formatting. + + Args: + cmd: openstack command to be executed in target. + + target: `crl.remotesession.remotesession.RemoteSession`_ target. + + Returns: + Nothing + + Raises: + OpenStackCliError: if remote *openstack* fails. + + .. _crl.remotesession.remotesession.RemoteSession: + https://crl-remotesession.readthedocs.io/en/latest + /crl.remotesession.remotesession.RemoteSession.html + """ + return self._create_runner(target).run(cmd).stdout + + def run_nohup(self, cmd, target='default'): + """ Run *openstack* in the remote target nohup mode in background and + return PID of the started process. + + Note: + This keyword is available only for targets initialized by + *Set Runner Target* keyword of *RemoteSession*. The version of + crl.interactivesessions must be at least as new as + crl.interactivesessions==1.0b4. + + Args: + cmd: openstack command to be executed in target. + + target: `crl.remotesession.remotesession.RemoteSession`_ target. + + Returns: + PID of the process running *cmd* in the target. + + Raises: + OpenStackCliError: if remote *openstack* fails. + + .. _crl.remotesession.remotesession.RemoteSession: + https://crl-remotesession.readthedocs.io/en/latest + /crl.remotesession.remotesession.RemoteSession.html + """ + return self._create_runner( + target, run=self._nohup_run).run_without_verification(cmd) + + def _create_runner(self, target, run=None): + run = self._run if run is None else run + r = _CliRunnerInTarget(run, self._get_target(target)) + r.set_cli(self._get_cli()) + return r + + def _get_target(self, target): + return self._create_default_target(target) + + def _create_default_target(self, target): + t = _DefaultTarget() + t.set_target(self._get_envtarget(target)) + return t + + def _get_envtarget(self, target): + return target if self._envname is None else '{target}.{envname}'.format( + target=target, + envname=self._envname) + + @classmethod + def _get_cli(cls): + return cls.__name__.lower() + + def _run(self, cmd, target): + with self._error_handling(cmd, target): + return self._remotesession.execute_command_in_target( + cmd, target=target) + + def _nohup_run(self, cmd, target): + runner = self._remotesession.get_remoterunner() + with self._error_handling(cmd, target): + return runner.execute_nohup_background_in_target(cmd, + target=target) + + @staticmethod + @contextmanager + def _error_handling(cmd, target): + try: + yield None + except Exception as e: # pylint: disable=broad-except + raise OpenStackCliError( + "{failed_msg}: {ecls}: {e}".format( + failed_msg=_FailedMessage(cmd, target), + ecls=e.__class__.__name__, + e=e)) + + +class Runner(OpenStack): + """The same as OpenStack_ but the remote command is executed withou prefix. + + **Note:** + + The Robot documentation of keywords is for *openstack* only. + + .. _OpenStack: crl.openstack.OpenStack.html + """ + + @staticmethod + def _get_cli(): + return '' diff --git a/libraries/openstackcli/service_show_neutron_expected_output.txt b/libraries/openstackcli/service_show_neutron_expected_output.txt new file mode 100644 index 0000000..37ab249 --- /dev/null +++ b/libraries/openstackcli/service_show_neutron_expected_output.txt @@ -0,0 +1,7 @@ +{ + "type": "network", + "enabled": true, + "description": "OpenStack Networking", + "name": "neutron", + "id": "a3bf29cfe238ab037ad235236a35323c" +} diff --git a/libraries/openstackcli/test_openstack.py b/libraries/openstackcli/test_openstack.py new file mode 100644 index 0000000..441352b --- /dev/null +++ b/libraries/openstackcli/test_openstack.py @@ -0,0 +1,150 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name +import os +import json +from collections import namedtuple +import pytest +from . openstackcli import ( + OpenStack, + OpenStackCliError) + + +class MockResult(namedtuple('MockResult', ['status', 'stdout', 'stderr'])): + def __str__(self): + return ', '.join([ + '{n}: {v!r}'.format(n=n, v=v) for n, v in self._asdict().items()]) + + +def get_output(): + return _get_content('service_show_neutron_expected_output.txt') + + +def _get_content(fname): + with open(os.path.join(os.path.dirname(__file__), fname)) as f: + return f.read() + + +@pytest.mark.parametrize('return_value', [ + MockResult(status=0, stdout=get_output(), stderr=''), + MockResult(status='0', stdout=get_output(), stderr='')]) +def test_openstack_run(cliwrapper, return_value): + cliwrapper.set_exec_func_and_return_value( + cliwrapper.remotesession.execute_command_in_target, + return_value=return_value) + cliwrapper.set_expected_cmd_postfix(' -f json') + + ret = cliwrapper.run_with_verify(cliwrapper.cli.run, 'service show neutron') + + runner_out = return_value.stdout + assert json.loads(runner_out) == ret + + +def test_run_ignore_output(cliwrapper): + return_value = MockResult(status=0, stdout='', stderr='') + cliwrapper.set_exec_func_and_return_value( + cliwrapper.remotesession.execute_command_in_target, + return_value=return_value) + + cliwrapper.run_with_verify(cliwrapper.cli.run_ignore_output, 'cmd') + + +def test_run_raw(cliwrapper): + return_value = MockResult(status=0, stdout='output', stderr='') + cliwrapper.set_exec_func_and_return_value( + cliwrapper.remotesession.execute_command_in_target, + return_value=return_value) + + assert cliwrapper.run_with_verify(cliwrapper.cli.run_raw, + 'cmd') == return_value.stdout + + +def test_openstack_run_nohup(cliwrapper): + runner = cliwrapper.remotesession.get_remoterunner.return_value + return_value = 'pid' + cliwrapper.set_exec_func_and_return_value( + runner.execute_nohup_background_in_target, + return_value=return_value) + + assert cliwrapper.run_with_verify(cliwrapper.cli.run_nohup, + 'cmd') == return_value + + +@pytest.fixture +def openstack(mock_remotesession): + n = OpenStack() + n.initialize(mock_remotesession) + return n + + +class RemoteException(Exception): + pass + + +def raise_remoteexception(): + raise RemoteException('message') + + +def test_openstack_run_runner_raises(mock_remotesession, + openstack): + + mock_remotesession.execute_command_in_target.side_effect = ( + lambda cmd, target: raise_remoteexception()) + + with pytest.raises(OpenStackCliError) as excinfo: + openstack.run('cmd') + + assert str(excinfo.value) == ( + "Remote command 'openstack --os-cloud default cmd -f json' " + "in target 'default' failed: RemoteException: message") + + +@pytest.fixture(params=[ + MockResult(status=1, stdout='out', stderr=''), + MockResult(status=0, stdout='out', stderr='err'), + MockResult(status='zero', stdout='out', stderr='')]) +def bad_mock_remotesession(request, + mock_remotesession): + mock_remotesession.execute_command_in_target.return_value = ( + request.param) + return mock_remotesession + + +def test_openstack_run_result_fail(bad_mock_remotesession, + methodfmt): + with pytest.raises(OpenStackCliError) as excinfo: + methodfmt.method('cmd') + + execute = bad_mock_remotesession.execute_command_in_target + assert str(excinfo.value) == ( + "Remote command 'openstack --os-cloud default cmd{fmt}' " + "in target 'default' failed: {return_value}".format( + fmt=methodfmt.fmt, + return_value=execute.return_value)) + + +def test_run_nohup_runner_raises(mock_remotesession, + openstack): + + runner = mock_remotesession.get_remoterunner.return_value + runner.execute_nohup_background_in_target.side_effect = ( + lambda cmd, target: raise_remoteexception()) + + with pytest.raises(OpenStackCliError) as excinfo: + openstack.run_nohup('cmd') + + assert str(excinfo.value) == ( + "Remote command 'openstack --os-cloud default cmd' " + "in target 'default' failed: RemoteException: message") diff --git a/libraries/pytestremotesession/__init__.py b/libraries/pytestremotesession/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libraries/pytestremotesession/fixtures.py b/libraries/pytestremotesession/fixtures.py new file mode 100644 index 0000000..7daa98f --- /dev/null +++ b/libraries/pytestremotesession/fixtures.py @@ -0,0 +1,29 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import mock + import pytest + from crl.remotesession.remotesession import RemoteSession + from crl.interactivesessions.remoterunner import RemoteRunner + + @pytest.fixture + def mock_remotesession(): + s = mock.create_autospec(RemoteSession) + s.get_remoterunner.return_value = mock.create_autospec(RemoteRunner) + return s + +except ImportError: + pass diff --git a/libraries/setup.py b/libraries/setup.py new file mode 100644 index 0000000..3590d58 --- /dev/null +++ b/libraries/setup.py @@ -0,0 +1,26 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import ( + setup, + find_packages) + + +setup( + name='cloudtaflibs', + install_requires=['crl.remotesession'], + entry_points={'pytest11': [ + 'pytestremotesession = pytestremotesession.fixtures']}, + include_package_data=True, + packages=find_packages()) diff --git a/requirements-minimal.txt b/requirements-minimal.txt new file mode 100644 index 0000000..fd68abd --- /dev/null +++ b/requirements-minimal.txt @@ -0,0 +1,3 @@ +crl.remotesession +crl.rfcli>=1.0 +pyyaml diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a1f9c70 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +asn1crypto==0.24.0 +bcrypt==3.1.6 +cffi==1.12.2 +crl.interactivesessions==1.1.4 +crl.remotescript==1.0.1 +crl.remotesession==1.2.1 +crl.rfcli==1.1.1 +crl.threadverify==1.0 +cryptography==2.6.1 +enum34==1.1.6 +future==0.17.1 +ipaddress==1.0.22 +paramiko==2.4.2 +pexpect==4.7.0 +ptyprocess==0.6.0 +pyasn1==0.4.5 +pycparser==2.19 +PyNaCl==1.3.0 +PyYAML==5.1 +robotframework==3.1.1 +six==1.12.0 diff --git a/resources/ssh.robot b/resources/ssh.robot new file mode 100644 index 0000000..40ec1cf --- /dev/null +++ b/resources/ssh.robot @@ -0,0 +1,69 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Defines targets to RemoteSession - library +... 'default' points to the active master via infra_external like +... the connections to the caas_master - nodes. +... Connection to the other nodes(caas_worker, storage) are done infra_internal +... via the active master. (in case the controller is also a storage the +... direct connection is used instead). + +Library crl.remotesession.remotesession.RemoteSession +... WITH NAME RemoteSession +Library cluster.cluster.Cluster +... WITH NAME Cluster + +*** Variables *** +@{ALL_MASTERS_IN_SYSTEM} @{EMPTY} +@{ALL_PURE_WORKERS_IN_SYSTEM} @{EMPTY} +@{ALL_PURE_STORAGES_IN_SYSTEM} @{EMPTY} +${IS_DPDK_IN_USE} ${False} + +*** Keywords *** + +Setup Connections + [Documentation] Setup targets for the RemoteSession Library for + ... for all nodes. + + Cluster.Initialize host=${RFCLI_TARGET_1.IP} + ... user=${RFCLI_TARGET_1.USER} + ... password=${RFCLI_TARGET_1.PASS} + + ${remotesession}= Get Library Instance RemoteSession + + Cluster.Initialize RemoteSession ${remotesession} + ${master}= Get Variable Value ${RFCLI_TARGET_1.MASTER_PROFILE} caas_master + ${worker}= Get Variable Value ${RFCLI_TARGET_1.WORKER_PROFILE} caas_worker + + ${masters}= Cluster.Get Hosts Containing ${master} + Set Global Variable ${ALL_MASTERS_IN_SYSTEM} ${masters} + + ${pure_storages}= Cluster.Get Hosts With Profiles storage + Set Global Variable ${ALL_PURE_STORAGES_IN_SYSTEM} ${pure_storages} + + ${pure_workers}= Cluster.Get Hosts With Profiles ${worker} + Set Global Variable ${ALL_PURE_WORKERS_IN_SYSTEM} ${pure_workers} + + ${is_dpdk_in_use}= Cluster.Is Dpdk + Set Global Variable ${IS_DPDK_IN_USE} ${is_dpdk_in_use} + + +Execute Command + [Arguments] ${CMD} ${NODE_NAME}=default ${CHECK_STDERR}=${True} + ${result}= RemoteSession.Execute Command In Target ${CMD} target=${NODE_NAME} + Should Be Equal As Integers ${result.status} 0 ${result.stderr} + Run Keyword If ${CHECK_STDERR} Should Be Empty ${result.stderr} + [Return] ${result.stdout} + diff --git a/rfcli-docker b/rfcli-docker new file mode 100755 index 0000000..8d25e27 --- /dev/null +++ b/rfcli-docker @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -u +set -x + +# TOX_ENV is tox environment which is to be executed. +if [ -z ${TOX_ENV+x} ]; then + TOX_ENV=rfcli +fi +SCRIPT_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-rec-cloudtaf-runner} + +$SCRIPT_DIR/rfcli-docker-build $DOCKER_IMAGE_TAG + +i=0 +ORIGINAL_ARGS=("$@") +NO_TARGET_CMD="" +while [ $i -lt ${#ORIGINAL_ARGS[@]} ]; do + arg=${ORIGINAL_ARGS[$i]} + if [ $arg == "-t" ]; then + target_index=$(expr $i + 1) + target_path=${ORIGINAL_ARGS[$target_index]} + i=$(expr $i + 2) + else + NO_TARGET_CMD="${NO_TARGET_CMD} ${arg}" + i=$(expr $i + 1) + fi +done + +cp $target_path targets +new_target_path=targets/$(basename $target_path) + +docker run --rm -t \ + --net="host" \ + --env DISPLAY \ + --env TOX_ENV \ + -v $(pwd):/tmp/rec-cloudtaf \ + -w /tmp/rec-cloudtaf \ + $DOCKER_IMAGE_TAG ./rfcli-tox-umask -t ${new_target_path} $NO_TARGET_CMD diff --git a/rfcli-docker-build b/rfcli-docker-build new file mode 100755 index 0000000..c9184b5 --- /dev/null +++ b/rfcli-docker-build @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -u + + +DOCKER_IMAGE_TAG=${1:-rec-cloudtaf-runner} + +docker build -t $DOCKER_IMAGE_TAG . diff --git a/rfcli-tox-umask b/rfcli-tox-umask new file mode 100755 index 0000000..9d9749c --- /dev/null +++ b/rfcli-tox-umask @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -u +umask 0000 +if [ -z ${TOX_ENV+x} ]; then + TOX_ENV=rfcli +fi + +tox -e ${TOX_ENV} -- $@ diff --git a/targets/README.rst b/targets/README.rst new file mode 100644 index 0000000..613376c --- /dev/null +++ b/targets/README.rst @@ -0,0 +1,49 @@ +.. + Copyright 2019 Nokia + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Targets +======= + +The target.ini files are copied by rfcli-docker script to this +directory. + +.. note: + + The target.ini files are not stored to git. + + +Target.ini files +---------------- + +The content of the target.ini files follows:: + + [target] + IP: + USER: + PASS: + +Target.ini files profiles properties 2019-Mar-26 +------------------------------------------------ + +Currently CaaS is not deployed and so neither caas_master nor caas_worker +service profiles exists. To solve this problem, additional properties +for such targets has to be added:: + + [target] + IP: + USER: + PASS: + MASTER_PROFILE: management + WORKER_PROFILE: base diff --git a/testcases/__init__.robot b/testcases/__init__.robot new file mode 100644 index 0000000..f19964a --- /dev/null +++ b/testcases/__init__.robot @@ -0,0 +1,32 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** + +Library cluster.cluster.Cluster WITH NAME Cluster +Library crl.remotesession.remotesession.RemoteSession + ... WITH NAME RemoteSession +Suite Setup Setup Suite +Suite Teardown Teardown Suite + +*** Keywords *** + +Setup Suite + ${master}= Get Variable Value ${RFCLI_TARGET_1.MASTER_PROFILE} caas_master + ${worker}= Get Variable Value ${RFCLI_TARGET_1.WORKER_PROFILE} caas_worker + Cluster.Set Profiles master=${master} worker=${worker} + +Teardown Suite + Run Keyword And Ignore Error Cluster.Delete Users + RemoteSession.Close diff --git a/testcases/smoke-tests/smoke-tests.robot b/testcases/smoke-tests/smoke-tests.robot new file mode 100644 index 0000000..a9a0b20 --- /dev/null +++ b/testcases/smoke-tests/smoke-tests.robot @@ -0,0 +1,115 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Library Collections +Library cluster.cluster.Cluster WITH NAME Cluster +Library crl.remotesession.remotesession.RemoteSession +... WITH NAME RemoteSession +Resource ssh.robot +Test Setup ssh.Setup Connections + +*** Keywords *** + +Validate Cluster + + :FOR ${node} IN @{ALL_MASTERS_IN_SYSTEM} + \ ${output}= Ssh.Execute Command hostname ${node} + \ Should Be Equal ${node} ${output} + + :FOR ${node} IN @{ALL_PURE_WORKERS_IN_SYSTEM} + \ ${output}= Ssh.Execute Command hostname ${node} + \ Should Be Equal ${node} ${output} + + :FOR ${node} IN @{ALL_PURE_STORAGES_IN_SYSTEM} + \ ${output}= Ssh.Execute Command hostname ${node} + \ Should Be Equal ${node} ${output} + + ${hosts}= Cluster.Get Hosts + :FOR ${node} IN @{hosts} + \ ${output}= Ssh.Execute Command hostname ${node.name} + \ Should Be Equal ${node.name} ${output} + + ${masters_len}= Get Length ${ALL_MASTERS_IN_SYSTEM} + + Should Be True ${masters_len} == 3 + ... Number of masters should be 3 not ${masters_len} + Verify Database + +Reboot Management VIP Node + RemoteSession.Execute Background Command In Target sleep 1; reboot + ... target=sudo-default + Sleep 1 + RemoteSession.Close + ssh.Setup Connections + +Stop Database + [Arguments] ${node} + ${result}= RemoteSession.Execute Command In Target + ... systemctl stop mariadb.service ${node} + Should Be Equal ${result.status} 0 ${result.stderr} + +Start Database + [Arguments] ${node} + ${result}= RemoteSession.Execute Command In Target + ... systemctl start mariadb.service ${node} + Should Be Equal ${result.status} 0 ${result.stderr} + +Verify Database + [Arguments] ${node}=sudo-default ${expected_cluster_size}=3 + ${out}= ssh.Execute Command mysql <<< "show status like 'wsrep_cluster_size';" + ... ${node} + Should match ${out} *wsrep_cluster_size*${expected_cluster_size}* + +*** Test Cases *** + +Verify Cluster Config Management + Validate Cluster + +Verify Cluster Config Management After Reboot Management VIP Node + Reboot Management VIP Node + Wait Until Keyword Succeeds 600s 1s Validate Cluster + +Verify Database Stop And Start + ${first_master}= Collections.Get From List ${ALL_MASTERS_IN_SYSTEM} 0 + Stop Database sudo-${first_master} + Wait Until Keyword Succeeds 6x 18s Verify Database sudo-${first_master} + ... expected_cluster_size=2 + Start Database sudo-${first_master} + Wait Until Keyword Succeeds 6x 18s Verify Database sudo-${first_master} + +Verify Create All And No Roles + [Documentation] Verify no_roles and all_roles user creation + ${all_roles}= Cluster.Create User With Roles all_roles + Log ${all_roles} + ${no_roles}= Cluster.Create User With Roles no_roles + Log ${no_roles} + +Verify SudoShells + [Documentation] Test SudoShells + ${hosts}= Cluster.Get Hosts + :FOR ${node} IN @{hosts} + \ ${result}= RemoteSession.Execute Command In Target whoami + \ ... target=sudo-${node.name} + \ Should Be Equal ${result.stdout} root + + ${result}= RemoteSession.Execute Command In Target whoami + ... target=sudo-default + Should Be Equal ${result.stdout} root + +Verify Remotescript Default + [Documentation] test target remotescript-default + ${result}= RemoteSession.Execute Command In Target echo -n out + ... target=remotescript-default + Should Be Equal ${result.stdout} out diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..a8965b6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,92 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[tox] +setupdir = {toxinidir}/libraries +envlist = py27, pylint, check-requirements + +[testenv] +setenv = + COVERAGE_FILE=.coverage.{envname} + +deps = + mock + pytest < 4.0 + pytest-pep8 + pytest-cov + more-itertools < 6.0.0 + -r{toxinidir}/requirements.txt + +commands = py.test -v \ + --basetemp={envtmpdir} \ + --pep8 \ + --cov . \ + --cov-branch \ + --cov-report term-missing \ + --cov-report html:coverage-html-{envname} \ + {posargs:.} + +[pytest] +cache_dir = .pytest-cache +pep8maxlinelength = 100 + +[testenv:pylint] +basepython = python2.7 +deps = + mock + pytest < 3.0 + pytest-pylint + pylint < 2.0 + -r{toxinidir}/requirements.txt + +commands = py.test -m pylint -v \ + --pylint \ + --pylint-rcfile={toxinidir}/.pylintrc \ + --ignore resources/system_testing/latency \ + {posargs:.} + + +[testenv:rfcli] +basepython = python2.7 + +passenv = + DISPLAY + HOME + +deps = + -r{toxinidir}/requirements.txt + +commands = + rfcli --pythonpath {toxinidir}:{toxinidir}/resources \ + --rfcli-no-pythonpath \ + {posargs} + +[testenv:check-requirements] +basepython = python2.7 +deps = + -r{toxinidir}/requirements.txt + more-itertools < 6.0.0 + requirements-tools==1.1.2 + pytest < 4.0 + +commands = check-requirements + +[testenv:freeze] +basepython = python2.7 + +deps = virtualenvrunner>=1.0 + +commands = create_virtualenv --recreate \ + --requirements {toxinidir}/requirements-minimal.txt \ + --save-freeze-path {toxinidir}/requirements.txt -- 2.16.6