X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fcloudtaf.git;a=blobdiff_plain;f=libraries%2Fcommon%2Fcommon_utils.py;fp=libraries%2Fcommon%2Fcommon_utils.py;h=058ccc291183771005d8af8f177f986cc5d11fee;hp=0000000000000000000000000000000000000000;hb=af5eb3ff36b92ab1d9c156ffa0391eadc73eb6ba;hpb=025a45508d009db84c34076fb4a668f712628d6d

diff --git a/libraries/common/common_utils.py b/libraries/common/common_utils.py
new file mode 100644
index 0000000..058ccc2
--- /dev/null
+++ b/libraries/common/common_utils.py
@@ -0,0 +1,374 @@
+import time
+import subprocess
+import os
+import re
+from datetime import datetime
+from datetime import timedelta
+import yaml
+from robot.api import logger
+from robot.libraries.BuiltIn import BuiltIn
+import ruamel.yaml
+from decorators_for_robot_functionalities import *
+from test_constants import *
+from users import *
+
+LOG_DIR = os.path.join(os.path.dirname(__file__))
+ex = BuiltIn().get_library_instance('execute_command')
+SSHLIB = ex.get_ssh_library_instance()
+STACK_INFOS = BuiltIn().get_library_instance('stack_infos')
+BuiltIn().import_library('pabot.PabotLib')
+PABOT = BuiltIn().get_library_instance('pabot.PabotLib')
+
+
+def keyword_runner(keywords, counter=0):
+    try:
+        BuiltIn().run_keyword(keywords[counter])
+    except Exception as err:
+        raise err
+    finally:
+        counter += 1
+        if len(keywords) > counter:
+            keyword_runner(keywords, counter)
+
+
+@robot_log
+def gather_logs(command, logfile_name, local_path):
+    remote_file_path = ROBOT_LOG_PATH + logfile_name
+    local_file_path = os.path.join(local_path, logfile_name)
+    ex.execute_unix_command_as_root("echo  -e '****** This is the output of: " +
+                                    command + " ****** \n' > " + remote_file_path)
+    ex.execute_unix_command_as_root(command + " >> " + remote_file_path)
+    ex.execute_unix_command_as_root("chmod 777 " + remote_file_path)
+    SSHLIB.get_file(remote_file_path, local_file_path)
+    ex.execute_unix_command_as_root("rm -f " + remote_file_path)
+
+
+@robot_log
+def gather_logs_from_remote(command, logfile_name, local_path, host, user={}):
+    if not user:
+        user = ex.get_default_user()
+    local_file_path = os.path.join(local_path, logfile_name)
+    remote_file_path = ROBOT_LOG_PATH + logfile_name
+    ex.execute_unix_command_on_remote_as_root("echo  -e '****** This is the output of: " +
+                                              command + " ****** \n' > " + remote_file_path, host, user, )
+    ex.execute_unix_command_on_remote_as_root(command + " >> " + remote_file_path, host, user)
+    transfer_file_from_remote(remote_file_path, remote_file_path, local_file_path, host, user)
+    ex.execute_unix_command_on_remote_as_root("rm -f " + remote_file_path, host, user)
+
+
+@robot_log
+def transfer_file_from_remote(remote_file_path, temp_file_path, local_file_path, host, user):
+    """"
+      This method is used to transfer a file  to the localhost, from a node other than the CRF_node_1.
+      :param remote_file_path: full file path on the remote node
+      :param temp_file_path: full file path on the CRF_node_1
+      :param local_file_path: full file path on the localhost
+      :param host: ip/hostname of the remote node
+      :param user: this user is used with the scp command
+    """
+    scp_command = "scp " + user['username'] + "@" + host + ":" + remote_file_path + " " + temp_file_path
+    SSHLIB.write(scp_command)
+    SSHLIB.read_until(host + "'s password:")
+    SSHLIB.write(user['password'])
+    SSHLIB.read_until(user['prompt'])
+    SSHLIB.get_file(temp_file_path, local_file_path)
+    ex.execute_unix_command_as_root("rm -f " + temp_file_path)
+
+
+def wait_for_healthy_kube_controller_manager():
+    wait_until = datetime.now() + timedelta(seconds=180)
+    command = "kubectl get componentstatus | grep controller-manager | grep Healthy | wc -l"
+    result = ex.execute_unix_command_as_root(command)
+    while (result < 1) and (datetime.now() < wait_until):
+        logger.info("datetime.now:" + str(datetime.now()))
+        logger.info("wait_until:" + str(wait_until))
+        logger.info("Controller-manager is not healthy yet, waiting...")
+        time.sleep(10)
+        result = ex.execute_unix_command_as_root(command)
+    if result < 1:
+        raise Exception("Controller-manager is not healthy!")
+
+
+@pabot_lock("health_check_1")
+@pabot_lock("modify_static_pod_config")
+def modify_static_pod_config(operation,
+                             manifest_file,
+                             flags):
+    """
+    This method inserts/removes the given flag list into the manifest file of a static pod.
+
+    :param manifest_file: manifest file name with extension present in /etc/kubernetes/manifests folder
+    :param flags: flags which will be given to the executed command in the container
+    :param operation: add or remove
+
+    """
+    crf_nodes = STACK_INFOS.get_crf_nodes()
+    if not crf_nodes:
+        logger.info("Nodes dictionary is empty, nothing to check.")
+        return
+    logger.info("adding flag to pod file")
+    for key in crf_nodes:
+        ex.execute_unix_command_on_remote_as_root("mv /etc/kubernetes/manifests/" +
+                                                  manifest_file + " /tmp/" + manifest_file, crf_nodes[key])
+        yaml_content = ruamel.yaml.round_trip_load(ex.execute_unix_command_on_remote_as_root("cat /tmp/" +
+                                                                                             manifest_file,
+                                                                                             crf_nodes[key]),
+                                                   preserve_quotes=True)
+        for actual_flag in flags:
+            operation(yaml_content, actual_flag)
+
+        yaml_content = ruamel.yaml.round_trip_dump(yaml_content, default_flow_style=False)
+    kube_controller_manager['obj_count'] = str(len(crf_nodes))
+    check_kubernetes_object(kube_controller_manager, test_kubernetes_object_not_available, timeout=300)
+
+    for key in crf_nodes:
+        ex.execute_unix_command_on_remote_as_root("echo \"" + yaml_content + "\" > /etc/kubernetes/manifests/" +
+                                                  manifest_file, crf_nodes[key])
+        ex.execute_unix_command_on_remote_as_root("rm -f /tmp/" + manifest_file, crf_nodes[key])
+    check_kubernetes_object(kube_controller_manager, test_kubernetes_object_available,
+                            additional_filter="Running", timeout=300)
+    wait_for_healthy_kube_controller_manager()
+
+
+@robot_log
+def add_flag_to_command(yaml_content, flag):
+    yaml_content["spec"]["containers"][0]["command"].append(flag)
+
+
+@robot_log
+def remove_flag_from_command(yaml_content, flag):
+    yaml_content["spec"]["containers"][0]["command"].remove(flag)
+
+
+@robot_log
+def helm_install(chart_name, release_name, values="registry_url={reg_url}".format(reg_url=reg)):
+    command = "helm install " + chart_name + " --name " + release_name
+    if values:
+        command += " --set " + values
+    ex.execute_unix_command(command, fail_on_non_zero_rc=False)
+    if helm_list(release_name) == '1':
+        logger.info(chart_name + " chart is successfully installed")
+    else:
+        raise Exception(chart_name + " chart install has failed.")
+
+
+@robot_log
+def helm_delete(release_name):
+    ex.execute_unix_command("helm delete " + release_name + " --purge ", delay="30s", fail_on_non_zero_rc=False)
+    if helm_list(release_name) == '0':
+        logger.info(release_name + " chart is successfully deleted")
+    else:
+        raise Exception(release_name + " chart delete has failed.")
+
+
+@robot_log
+def helm_list(release_name, add_check_arg=''):
+    grep_arg = 'grep -w {}'.format(release_name)
+    if add_check_arg != '':
+        grep_arg += '| grep -w {}'.format(add_check_arg)
+    command = "helm list --all | {} | wc -l".format(grep_arg)
+    stdout, _ = ex.execute_unix_command(command, fail_on_non_zero_rc=False)
+    return stdout.strip()
+
+
+@robot_log
+def check_kubernetes_object(kube_object, tester_function, additional_filter=".*", timeout=0, delay=0):
+    """"
+    This method executes kubectl get command with the given args, filters the output and checks the result with
+      the given tester_function.
+      :param kube_object: a dictionary, it represents a kubernetes objects,
+                          obj_type, obj_name, namespace, obj_count keys are required.
+      :param tester_function: this functoin checks the result and waits for the expected result
+                              - kubernetes object exists or not - to happen in a given time
+      :param additional_filter: use this regexp to filter further the results
+      :param timeout: wait <timeout> seconds for the result
+      :param delay: wait <delay> seconds before tester command
+    """""
+    command = "kubectl get {object} -n {ns_arg} 2>/dev/null | grep -w {name} | grep -E '{grep_arg}' | wc -l"
+    command = command.format(object=kube_object['obj_type'], name=kube_object['obj_name'],
+                             ns_arg=kube_object['namespace'], grep_arg=additional_filter)
+    tester_function(kube_object, timeout, command, delay)
+
+
+@robot_log
+def is_result_expected_within_given_time(command, expected_result, timeout, delay=0):
+    time.sleep(delay)
+    result = ex.execute_unix_command(command)
+    if result == expected_result:
+        return True
+    wait_until = datetime.now() + timedelta(seconds=timeout)
+    while result != expected_result and (datetime.now() < wait_until):
+        logger.info("datetime.now:" + str(datetime.now()))
+        logger.info("wait_until:" + str(wait_until))
+        logger.info("expected result: " + expected_result)
+        logger.info("result: " + result)
+        time.sleep(1)
+        result = ex.execute_unix_command(command)
+        if result == expected_result:
+            return True
+    return False
+
+
+def test_kubernetes_object_quality(kube_object, expected_result, filter=".*", timeout=30, delay=0):
+    tester_command = "kubectl get " + kube_object['obj_type'] + " --all-namespaces | grep -w " + \
+                     kube_object['obj_name'] + " | grep -E '" + filter + "' | wc -l"
+    res = is_result_expected_within_given_time(tester_command, expected_result, timeout, delay)
+    if not res:
+        log_command = "kubectl get " + kube_object['obj_type'] + " --all-namespaces | grep -w " + \
+                      kube_object['obj_name']
+        res = ex.execute_unix_command(log_command)
+        ex.execute_unix_command("kubectl describe " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " -n " +
+                                kube_object['namespace'])
+        raise Exception("Not " + kube_object['obj_count'] + " " + kube_object['obj_type'] + " " +
+                        kube_object['obj_name'] + " is in expected (" + filter + ") state:" + res)
+    logger.console(kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] +
+                   " is in expected (" + filter + ") state.")
+
+
+def test_kubernetes_object_available(kube_object, timeout, tester_command, delay=0):
+    res = is_result_expected_within_given_time(tester_command, kube_object['obj_count'], timeout=timeout, delay=delay)
+    if not res:
+        describe_command = "kubectl describe " + kube_object['obj_type'] + " -n " + \
+                           kube_object['namespace'] + " " + kube_object['obj_name']
+        ex.execute_unix_command(describe_command, fail_on_non_zero_rc=False)
+        raise Exception("Not " + kube_object['obj_count'] + " " + kube_object['obj_type'] + " " +
+                        kube_object['obj_name'] + " is running!")
+    logger.console(kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] +
+                   " is running, as expected!")
+
+
+def test_kubernetes_object_not_available(kube_object, timeout, tester_command, delay=0):
+    res = is_result_expected_within_given_time(tester_command, expected_result="0", timeout=timeout, delay=delay)
+    if not res:
+        describe_command = "kubectl describe " + kube_object['obj_type'] + " -n " + \
+                           kube_object['namespace'] + " " + kube_object['obj_name']
+        ex.execute_unix_command(describe_command, fail_on_non_zero_rc=False)
+        raise Exception("At least 1 " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " still exists!")
+    logger.console(kube_object['obj_type'] + " " + kube_object['obj_name'] + " does not exist, as expected!")
+
+
+def is_node_under_pressure(nodeslog):
+    return bool(nodeslog.find("pressure") != -1)
+
+
+def wait_if_pressure(timeout=pressure_default_timeout):
+    wait_until = datetime.now() + timedelta(seconds=timeout)
+    command = "kubectl get nodes -o json | jq '.items[] | \"\(.metadata.name) \(.spec.taints)\"'"
+    nodeslog = ex.execute_unix_command_as_root(command)
+    while (is_node_under_pressure(nodeslog)) and (datetime.now() < wait_until):
+        logger.info("datetime.now:" + str(datetime.now()))
+        logger.info("wait_until:" + str(wait_until))
+        logger.info("Node is under pressure found: " + nodeslog)
+        time.sleep(10)
+        nodeslog = ex.execute_unix_command_as_root(command)
+    if is_node_under_pressure(nodeslog):
+        raise Exception("Node pressure not resolved in time.")
+    else:
+        logger.info(nodeslog)
+
+
+@robot_log
+def check_url_running(filename, url):
+    command = "curl -s {url} > /dev/null ; echo -n $?"
+    result = ex.execute_unix_command_as_root(command.format(url=url))
+    if result == "0":
+        logger.console("{url} is running!".format(url=url))
+    else:
+        gather_logs("curl -s {url}".format(url=url), filename, LOG_DIR)
+        raise Exception("{url} is not running !".format(url=url))
+
+
+@robot_log
+def subprocess_cmd(command):
+    return subprocess.check_output(command, shell=True).strip()
+
+
+@robot_log
+def put_file(local_script_path, remote_script_path, permissions="777", user=root['username'], group=root['username']):
+    ex.get_ssh_library_instance().put_file(local_script_path, remote_script_path, permissions)
+    head, tail = os.path.split(remote_script_path)
+    command = 'ls -l ' + head + ' | grep ' + tail + ' | wc -l'
+    res = is_result_expected_within_given_time(command, expected_result="1", timeout=5)
+    if not res:
+        raise Exception("File not found at " + remote_script_path + "!")
+    ex.execute_unix_command_as_root('chgrp ' + group + ' ' + remote_script_path)
+    ex.execute_unix_command_as_root('chown ' + user + ' ' + remote_script_path)
+
+
+@robot_log
+def get_helm_chart_content(chart_name):
+    ex.execute_unix_command("helm fetch " + chart_name + " --untar --untardir /tmp")
+    return ex.execute_unix_command("ls /tmp/" + chart_name.split('/')[1] +
+                                   "/templates | awk -F . '{print $1}'").split('\r\n')
+
+
+@robot_log
+def get_cpupools():
+    node_map = {}
+    node_list = ex.execute_unix_command("kubectl get nodes -L=nodename | awk '{print $6}'| tail -n +2")
+    cmap_str = ex.execute_unix_command("kubectl get configmap -n kube-system {cm} -o yaml"
+                                       .format(cm=cpu_pooling_cm_name))
+    for nodename in node_list.splitlines():  # pylint: disable=too-many-nested-blocks
+        yamldict = yaml.load(cmap_str)
+        for key in yamldict['data']:
+            if nodename in yamldict['data'][key]:
+                worker_yaml = yaml.load(yamldict['data'][key])
+                pool_dict = {}
+                if worker_yaml['pools']:
+                    for pool in worker_yaml['pools']:
+                        pool_str = worker_yaml['pools'][pool]['cpus']
+                        pool_list = []
+                        for sub_list in pool_str.split(','):
+                            pool_list = pool_list + ([int(sub_list)] if '-' not in sub_list else
+                                                     range(int(sub_list.split('-')[0]),
+                                                           int(sub_list.split('-')[1]) + 1))
+                        pool_dict[pool] = pool_list
+                node_map[nodename] = pool_dict
+    return node_map
+
+
+@robot_log
+def get_cpu_allowed_list_from_pod(pod_name):
+    bash_command = "cat /proc/1/status | grep Cpus_allowed_list"
+    result = ex.execute_unix_command("kubectl exec `kubectl get pod | grep {0} | "
+                                     "awk '{{print $1}}'` -- {1}".format(pod_name, bash_command))
+    pool_list = []
+    for cpu in result.split(':')[1].split(','):
+        pool_list = pool_list + ([int(cpu)] if '-' not in cpu else range(int(cpu.split('-')[0]),
+                                                                         int(cpu.split('-')[1]) + 1))
+    return pool_list
+
+
+@robot_log
+def allowed_cpus_is_in_cpu_pool(allowed_cpus, cpu_pool):
+    for allowed in allowed_cpus:
+        if allowed not in cpu_pool:
+            return False
+    return True
+
+
+def decide_nodename():
+    nodename = 'caas_worker1'
+    command = "kubectl get node -L=nodename | awk {{'print $6'}} | tail -n +2"
+    node_names = ex.execute_unix_command(command)
+    if nodename not in node_names:
+        return node_names.splitlines()[0]
+    return nodename
+
+
+@robot_log
+def determine_accurate_running_time_of_obj(object_type, object_name):
+    hours = mins = secs = 0
+    cmd = "kubectl get {obj_type} --all-namespaces --no-headers=true | grep {obj_name} | awk '{{print $NF}}'" \
+        .format(obj_type=object_type, obj_name=object_name)
+    resp = ex.execute_unix_command(cmd)
+    pod_time = re.findall(r'\d{0,2}h|\d{0,3}m|\d{1,3}s', resp)
+    for t in pod_time:
+        if t[-1] == 'h':
+            hours = int(t[:-1])
+        elif t[-1] == 'm':
+            mins = int(t[:-1])
+        elif t[-1] == 's':
+            secs = int(t[:-1])
+
+    return datetime.now() - timedelta(hours=hours, minutes=mins, seconds=secs)