libraries/common/common_utils.py

   1 import time
   2 import subprocess
   3 import os
   4 import re
   5 from datetime import datetime
   6 from datetime import timedelta
   7 import yaml
   8 from robot.api import logger
   9 from robot.libraries.BuiltIn import BuiltIn
  10 import ruamel.yaml
  11 from decorators_for_robot_functionalities import *
  12 from test_constants import *
  13 from users import *
  14
  15 LOG_DIR = os.path.join(os.path.dirname(__file__))
  16 ex = BuiltIn().get_library_instance('execute_command')
  17 SSHLIB = ex.get_ssh_library_instance()
  18 STACK_INFOS = BuiltIn().get_library_instance('stack_infos')
  19 BuiltIn().import_library('pabot.PabotLib')
  20 PABOT = BuiltIn().get_library_instance('pabot.PabotLib')
  21
  22
  23 def keyword_runner(keywords, counter=0):
  24     try:
  25         BuiltIn().run_keyword(keywords[counter])
  26     except Exception as err:
  27         raise err
  28     finally:
  29         counter += 1
  30         if len(keywords) > counter:
  31             keyword_runner(keywords, counter)
  32
  33
  34 @robot_log
  35 def gather_logs(command, logfile_name, local_path):
  36     remote_file_path = ROBOT_LOG_PATH + logfile_name
  37     local_file_path = os.path.join(local_path, logfile_name)
  38     ex.execute_unix_command_as_root("echo  -e '****** This is the output of: " +
  39                                     command + " ****** \n' > " + remote_file_path)
  40     ex.execute_unix_command_as_root(command + " >> " + remote_file_path)
  41     ex.execute_unix_command_as_root("chmod 777 " + remote_file_path)
  42     SSHLIB.get_file(remote_file_path, local_file_path)
  43     ex.execute_unix_command_as_root("rm -f " + remote_file_path)
  44
  45
  46 @robot_log
  47 def gather_logs_from_remote(command, logfile_name, local_path, host, user={}):
  48     if not user:
  49         user = ex.get_default_user()
  50     local_file_path = os.path.join(local_path, logfile_name)
  51     remote_file_path = ROBOT_LOG_PATH + logfile_name
  52     ex.execute_unix_command_on_remote_as_root("echo  -e '****** This is the output of: " +
  53                                               command + " ****** \n' > " + remote_file_path, host, user, )
  54     ex.execute_unix_command_on_remote_as_root(command + " >> " + remote_file_path, host, user)
  55     transfer_file_from_remote(remote_file_path, remote_file_path, local_file_path, host, user)
  56     ex.execute_unix_command_on_remote_as_root("rm -f " + remote_file_path, host, user)
  57
  58
  59 @robot_log
  60 def transfer_file_from_remote(remote_file_path, temp_file_path, local_file_path, host, user):
  61     """"
  62       This method is used to transfer a file  to the localhost, from a node other than the CRF_node_1.
  63       :param remote_file_path: full file path on the remote node
  64       :param temp_file_path: full file path on the CRF_node_1
  65       :param local_file_path: full file path on the localhost
  66       :param host: ip/hostname of the remote node
  67       :param user: this user is used with the scp command
  68     """
  69     scp_command = "scp " + user['username'] + "@" + host + ":" + remote_file_path + " " + temp_file_path
  70     SSHLIB.write(scp_command)
  71     SSHLIB.read_until(host + "'s password:")
  72     SSHLIB.write(user['password'])
  73     SSHLIB.read_until(user['prompt'])
  74     SSHLIB.get_file(temp_file_path, local_file_path)
  75     ex.execute_unix_command_as_root("rm -f " + temp_file_path)
  76
  77
  78 def wait_for_healthy_kube_controller_manager():
  79     wait_until = datetime.now() + timedelta(seconds=180)
  80     command = "kubectl get componentstatus | grep controller-manager | grep Healthy | wc -l"
  81     result = ex.execute_unix_command_as_root(command)
  82     while (result < 1) and (datetime.now() < wait_until):
  83         logger.info("datetime.now:" + str(datetime.now()))
  84         logger.info("wait_until:" + str(wait_until))
  85         logger.info("Controller-manager is not healthy yet, waiting...")
  86         time.sleep(10)
  87         result = ex.execute_unix_command_as_root(command)
  88     if result < 1:
  89         raise Exception("Controller-manager is not healthy!")
  90
  91
  92 @pabot_lock("health_check_1")
  93 @pabot_lock("modify_static_pod_config")
  94 def modify_static_pod_config(operation,
  95                              manifest_file,
  96                              flags):
  97     """
  98     This method inserts/removes the given flag list into the manifest file of a static pod.
  99
 100     :param manifest_file: manifest file name with extension present in /etc/kubernetes/manifests folder
 101     :param flags: flags which will be given to the executed command in the container
 102     :param operation: add or remove
 103
 104     """
 105     crf_nodes = STACK_INFOS.get_crf_nodes()
 106     if not crf_nodes:
 107         logger.info("Nodes dictionary is empty, nothing to check.")
 108         return
 109     logger.info("adding flag to pod file")
 110     for key in crf_nodes:
 111         ex.execute_unix_command_on_remote_as_root("mv /etc/kubernetes/manifests/" +
 112                                                   manifest_file + " /tmp/" + manifest_file, crf_nodes[key])
 113         yaml_content = ruamel.yaml.round_trip_load(ex.execute_unix_command_on_remote_as_root("cat /tmp/" +
 114                                                                                              manifest_file,
 115                                                                                              crf_nodes[key]),
 116                                                    preserve_quotes=True)
 117         for actual_flag in flags:
 118             operation(yaml_content, actual_flag)
 119
 120         yaml_content = ruamel.yaml.round_trip_dump(yaml_content, default_flow_style=False)
 121     kube_controller_manager['obj_count'] = str(len(crf_nodes))
 122     check_kubernetes_object(kube_controller_manager, test_kubernetes_object_not_available, timeout=300)
 123
 124     for key in crf_nodes:
 125         ex.execute_unix_command_on_remote_as_root("echo \"" + yaml_content + "\" > /etc/kubernetes/manifests/" +
 126                                                   manifest_file, crf_nodes[key])
 127         ex.execute_unix_command_on_remote_as_root("rm -f /tmp/" + manifest_file, crf_nodes[key])
 128     check_kubernetes_object(kube_controller_manager, test_kubernetes_object_available,
 129                             additional_filter="Running", timeout=300)
 130     wait_for_healthy_kube_controller_manager()
 131
 132
 133 @robot_log
 134 def add_flag_to_command(yaml_content, flag):
 135     yaml_content["spec"]["containers"][0]["command"].append(flag)
 136
 137
 138 @robot_log
 139 def remove_flag_from_command(yaml_content, flag):
 140     yaml_content["spec"]["containers"][0]["command"].remove(flag)
 141
 142
 143 @robot_log
 144 def helm_install(chart_name, release_name, values="registry_url={reg_url}".format(reg_url=reg)):
 145     command = "helm install " + chart_name + " --name " + release_name
 146     if values:
 147         command += " --set " + values
 148     ex.execute_unix_command(command, fail_on_non_zero_rc=False)
 149     if helm_list(release_name) == '1':
 150         logger.info(chart_name + " chart is successfully installed")
 151     else:
 152         raise Exception(chart_name + " chart install has failed.")
 153
 154
 155 @robot_log
 156 def helm_delete(release_name):
 157     ex.execute_unix_command("helm delete " + release_name + " --purge ", delay="30s", fail_on_non_zero_rc=False)
 158     if helm_list(release_name) == '0':
 159         logger.info(release_name + " chart is successfully deleted")
 160     else:
 161         raise Exception(release_name + " chart delete has failed.")
 162
 163
 164 @robot_log
 165 def helm_list(release_name, add_check_arg=''):
 166     grep_arg = 'grep -w {}'.format(release_name)
 167     if add_check_arg != '':
 168         grep_arg += '| grep -w {}'.format(add_check_arg)
 169     command = "helm list --all | {} | wc -l".format(grep_arg)
 170     stdout, _ = ex.execute_unix_command(command, fail_on_non_zero_rc=False)
 171     return stdout.strip()
 172
 173
 174 @robot_log
 175 def check_kubernetes_object(kube_object, tester_function, additional_filter=".*", timeout=0, delay=0):
 176     """"
 177     This method executes kubectl get command with the given args, filters the output and checks the result with
 178       the given tester_function.
 179       :param kube_object: a dictionary, it represents a kubernetes objects,
 180                           obj_type, obj_name, namespace, obj_count keys are required.
 181       :param tester_function: this functoin checks the result and waits for the expected result
 182                               - kubernetes object exists or not - to happen in a given time
 183       :param additional_filter: use this regexp to filter further the results
 184       :param timeout: wait <timeout> seconds for the result
 185       :param delay: wait <delay> seconds before tester command
 186     """""
 187     command = "kubectl get {object} -n {ns_arg} 2>/dev/null | grep -w {name} | grep -E '{grep_arg}' | wc -l"
 188     command = command.format(object=kube_object['obj_type'], name=kube_object['obj_name'],
 189                              ns_arg=kube_object['namespace'], grep_arg=additional_filter)
 190     tester_function(kube_object, timeout, command, delay)
 191
 192
 193 @robot_log
 194 def is_result_expected_within_given_time(command, expected_result, timeout, delay=0):
 195     time.sleep(delay)
 196     result = ex.execute_unix_command(command)
 197     if result == expected_result:
 198         return True
 199     wait_until = datetime.now() + timedelta(seconds=timeout)
 200     while result != expected_result and (datetime.now() < wait_until):
 201         logger.info("datetime.now:" + str(datetime.now()))
 202         logger.info("wait_until:" + str(wait_until))
 203         logger.info("expected result: " + expected_result)
 204         logger.info("result: " + result)
 205         time.sleep(1)
 206         result = ex.execute_unix_command(command)
 207         if result == expected_result:
 208             return True
 209     return False
 210
 211
 212 def test_kubernetes_object_quality(kube_object, expected_result, filter=".*", timeout=30, delay=0):
 213     tester_command = "kubectl get " + kube_object['obj_type'] + " --all-namespaces | grep -w " + \
 214                      kube_object['obj_name'] + " | grep -E '" + filter + "' | wc -l"
 215     res = is_result_expected_within_given_time(tester_command, expected_result, timeout, delay)
 216     if not res:
 217         log_command = "kubectl get " + kube_object['obj_type'] + " --all-namespaces | grep -w " + \
 218                       kube_object['obj_name']
 219         res = ex.execute_unix_command(log_command)
 220         ex.execute_unix_command("kubectl describe " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " -n " +
 221                                 kube_object['namespace'])
 222         raise Exception("Not " + kube_object['obj_count'] + " " + kube_object['obj_type'] + " " +
 223                         kube_object['obj_name'] + " is in expected (" + filter + ") state:" + res)
 224     logger.console(kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] +
 225                    " is in expected (" + filter + ") state.")
 226
 227
 228 def test_kubernetes_object_available(kube_object, timeout, tester_command, delay=0):
 229     res = is_result_expected_within_given_time(tester_command, kube_object['obj_count'], timeout=timeout, delay=delay)
 230     if not res:
 231         describe_command = "kubectl describe " + kube_object['obj_type'] + " -n " + \
 232                            kube_object['namespace'] + " " + kube_object['obj_name']
 233         ex.execute_unix_command(describe_command, fail_on_non_zero_rc=False)
 234         raise Exception("Not " + kube_object['obj_count'] + " " + kube_object['obj_type'] + " " +
 235                         kube_object['obj_name'] + " is running!")
 236     logger.console(kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] +
 237                    " is running, as expected!")
 238
 239
 240 def test_kubernetes_object_not_available(kube_object, timeout, tester_command, delay=0):
 241     res = is_result_expected_within_given_time(tester_command, expected_result="0", timeout=timeout, delay=delay)
 242     if not res:
 243         describe_command = "kubectl describe " + kube_object['obj_type'] + " -n " + \
 244                            kube_object['namespace'] + " " + kube_object['obj_name']
 245         ex.execute_unix_command(describe_command, fail_on_non_zero_rc=False)
 246         raise Exception("At least 1 " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " still exists!")
 247     logger.console(kube_object['obj_type'] + " " + kube_object['obj_name'] + " does not exist, as expected!")
 248
 249
 250 def is_node_under_pressure(nodeslog):
 251     return bool(nodeslog.find("pressure") != -1)
 252
 253
 254 def wait_if_pressure(timeout=pressure_default_timeout):
 255     wait_until = datetime.now() + timedelta(seconds=timeout)
 256     command = "kubectl get nodes -o json | jq '.items[] | \"\(.metadata.name) \(.spec.taints)\"'"
 257     nodeslog = ex.execute_unix_command_as_root(command)
 258     while (is_node_under_pressure(nodeslog)) and (datetime.now() < wait_until):
 259         logger.info("datetime.now:" + str(datetime.now()))
 260         logger.info("wait_until:" + str(wait_until))
 261         logger.info("Node is under pressure found: " + nodeslog)
 262         time.sleep(10)
 263         nodeslog = ex.execute_unix_command_as_root(command)
 264     if is_node_under_pressure(nodeslog):
 265         raise Exception("Node pressure not resolved in time.")
 266     else:
 267         logger.info(nodeslog)
 268
 269
 270 @robot_log
 271 def check_url_running(filename, url):
 272     command = "curl -s {url} > /dev/null ; echo -n $?"
 273     result = ex.execute_unix_command_as_root(command.format(url=url))
 274     if result == "0":
 275         logger.console("{url} is running!".format(url=url))
 276     else:
 277         gather_logs("curl -s {url}".format(url=url), filename, LOG_DIR)
 278         raise Exception("{url} is not running !".format(url=url))
 279
 280
 281 @robot_log
 282 def subprocess_cmd(command):
 283     return subprocess.check_output(command, shell=True).strip()
 284
 285
 286 @robot_log
 287 def put_file(local_script_path, remote_script_path, permissions="777", user=root['username'], group=root['username']):
 288     ex.get_ssh_library_instance().put_file(local_script_path, remote_script_path, permissions)
 289     head, tail = os.path.split(remote_script_path)
 290     command = 'ls -l ' + head + ' | grep ' + tail + ' | wc -l'
 291     res = is_result_expected_within_given_time(command, expected_result="1", timeout=5)
 292     if not res:
 293         raise Exception("File not found at " + remote_script_path + "!")
 294     ex.execute_unix_command_as_root('chgrp ' + group + ' ' + remote_script_path)
 295     ex.execute_unix_command_as_root('chown ' + user + ' ' + remote_script_path)
 296
 297
 298 @robot_log
 299 def get_helm_chart_content(chart_name):
 300     ex.execute_unix_command("helm fetch " + chart_name + " --untar --untardir /tmp")
 301     return ex.execute_unix_command("ls /tmp/" + chart_name.split('/')[1] +
 302                                    "/templates | awk -F . '{print $1}'").split('\r\n')
 303
 304
 305 @robot_log
 306 def get_cpupools():
 307     node_map = {}
 308     node_list = ex.execute_unix_command("kubectl get nodes -L=nodename | awk '{print $6}'| tail -n +2")
 309     cmap_str = ex.execute_unix_command("kubectl get configmap -n kube-system {cm} -o yaml"
 310                                        .format(cm=cpu_pooling_cm_name))
 311     for nodename in node_list.splitlines():  # pylint: disable=too-many-nested-blocks
 312         yamldict = yaml.load(cmap_str)
 313         for key in yamldict['data']:
 314             if nodename in yamldict['data'][key]:
 315                 worker_yaml = yaml.load(yamldict['data'][key])
 316                 pool_dict = {}
 317                 if worker_yaml['pools']:
 318                     for pool in worker_yaml['pools']:
 319                         pool_str = worker_yaml['pools'][pool]['cpus']
 320                         pool_list = []
 321                         for sub_list in pool_str.split(','):
 322                             pool_list = pool_list + ([int(sub_list)] if '-' not in sub_list else
 323                                                      range(int(sub_list.split('-')[0]),
 324                                                            int(sub_list.split('-')[1]) + 1))
 325                         pool_dict[pool] = pool_list
 326                 node_map[nodename] = pool_dict
 327     return node_map
 328
 329
 330 @robot_log
 331 def get_cpu_allowed_list_from_pod(pod_name):
 332     bash_command = "cat /proc/1/status | grep Cpus_allowed_list"
 333     result = ex.execute_unix_command("kubectl exec `kubectl get pod | grep {0} | "
 334                                      "awk '{{print $1}}'` -- {1}".format(pod_name, bash_command))
 335     pool_list = []
 336     for cpu in result.split(':')[1].split(','):
 337         pool_list = pool_list + ([int(cpu)] if '-' not in cpu else range(int(cpu.split('-')[0]),
 338                                                                          int(cpu.split('-')[1]) + 1))
 339     return pool_list
 340
 341
 342 @robot_log
 343 def allowed_cpus_is_in_cpu_pool(allowed_cpus, cpu_pool):
 344     for allowed in allowed_cpus:
 345         if allowed not in cpu_pool:
 346             return False
 347     return True
 348
 349
 350 def decide_nodename():
 351     nodename = 'caas_worker1'
 352     command = "kubectl get node -L=nodename | awk {{'print $6'}} | tail -n +2"
 353     node_names = ex.execute_unix_command(command)
 354     if nodename not in node_names:
 355         return node_names.splitlines()[0]
 356     return nodename
 357
 358
 359 @robot_log
 360 def determine_accurate_running_time_of_obj(object_type, object_name):
 361     hours = mins = secs = 0
 362     cmd = "kubectl get {obj_type} --all-namespaces --no-headers=true | grep {obj_name} | awk '{{print $NF}}'" \
 363         .format(obj_type=object_type, obj_name=object_name)
 364     resp = ex.execute_unix_command(cmd)
 365     pod_time = re.findall(r'\d{0,2}h|\d{0,3}m|\d{1,3}s', resp)
 366     for t in pod_time:
 367         if t[-1] == 'h':
 368             hours = int(t[:-1])
 369         elif t[-1] == 'm':
 370             mins = int(t[:-1])
 371         elif t[-1] == 's':
 372             secs = int(t[:-1])
 373
 374     return datetime.now() - timedelta(hours=hours, minutes=mins, seconds=secs)