import time import subprocess import os import re from datetime import datetime from datetime import timedelta import yaml from robot.api import logger from robot.libraries.BuiltIn import BuiltIn import ruamel.yaml from decorators_for_robot_functionalities import * from test_constants import * from users import * LOG_DIR = os.path.join(os.path.dirname(__file__)) ex = BuiltIn().get_library_instance('execute_command') SSHLIB = ex.get_ssh_library_instance() STACK_INFOS = BuiltIn().get_library_instance('stack_infos') BuiltIn().import_library('pabot.PabotLib') PABOT = BuiltIn().get_library_instance('pabot.PabotLib') def keyword_runner(keywords, counter=0): try: BuiltIn().run_keyword(keywords[counter]) except Exception as err: raise err finally: counter += 1 if len(keywords) > counter: keyword_runner(keywords, counter) @robot_log def gather_logs(command, logfile_name, local_path): remote_file_path = ROBOT_LOG_PATH + logfile_name local_file_path = os.path.join(local_path, logfile_name) ex.execute_unix_command_as_root("echo -e '****** This is the output of: " + command + " ****** \n' > " + remote_file_path) ex.execute_unix_command_as_root(command + " >> " + remote_file_path) ex.execute_unix_command_as_root("chmod 777 " + remote_file_path) SSHLIB.get_file(remote_file_path, local_file_path) ex.execute_unix_command_as_root("rm -f " + remote_file_path) @robot_log def gather_logs_from_remote(command, logfile_name, local_path, host, user={}): if not user: user = ex.get_default_user() local_file_path = os.path.join(local_path, logfile_name) remote_file_path = ROBOT_LOG_PATH + logfile_name ex.execute_unix_command_on_remote_as_root("echo -e '****** This is the output of: " + command + " ****** \n' > " + remote_file_path, host, user, ) ex.execute_unix_command_on_remote_as_root(command + " >> " + remote_file_path, host, user) transfer_file_from_remote(remote_file_path, remote_file_path, local_file_path, host, user) ex.execute_unix_command_on_remote_as_root("rm -f " + remote_file_path, host, user) @robot_log def transfer_file_from_remote(remote_file_path, temp_file_path, local_file_path, host, user): """" This method is used to transfer a file to the localhost, from a node other than the CRF_node_1. :param remote_file_path: full file path on the remote node :param temp_file_path: full file path on the CRF_node_1 :param local_file_path: full file path on the localhost :param host: ip/hostname of the remote node :param user: this user is used with the scp command """ scp_command = "scp " + user['username'] + "@" + host + ":" + remote_file_path + " " + temp_file_path SSHLIB.write(scp_command) SSHLIB.read_until(host + "'s password:") SSHLIB.write(user['password']) SSHLIB.read_until(user['prompt']) SSHLIB.get_file(temp_file_path, local_file_path) ex.execute_unix_command_as_root("rm -f " + temp_file_path) def wait_for_healthy_kube_controller_manager(): wait_until = datetime.now() + timedelta(seconds=180) command = "kubectl get componentstatus | grep controller-manager | grep Healthy | wc -l" result = ex.execute_unix_command_as_root(command) while (result < 1) and (datetime.now() < wait_until): logger.info("datetime.now:" + str(datetime.now())) logger.info("wait_until:" + str(wait_until)) logger.info("Controller-manager is not healthy yet, waiting...") time.sleep(10) result = ex.execute_unix_command_as_root(command) if result < 1: raise Exception("Controller-manager is not healthy!") @pabot_lock("health_check_1") @pabot_lock("modify_static_pod_config") def modify_static_pod_config(operation, manifest_file, flags): """ This method inserts/removes the given flag list into the manifest file of a static pod. :param manifest_file: manifest file name with extension present in /etc/kubernetes/manifests folder :param flags: flags which will be given to the executed command in the container :param operation: add or remove """ crf_nodes = STACK_INFOS.get_crf_nodes() if not crf_nodes: logger.info("Nodes dictionary is empty, nothing to check.") return logger.info("adding flag to pod file") for key in crf_nodes: ex.execute_unix_command_on_remote_as_root("mv /etc/kubernetes/manifests/" + manifest_file + " /tmp/" + manifest_file, crf_nodes[key]) yaml_content = ruamel.yaml.round_trip_load(ex.execute_unix_command_on_remote_as_root("cat /tmp/" + manifest_file, crf_nodes[key]), preserve_quotes=True) for actual_flag in flags: operation(yaml_content, actual_flag) yaml_content = ruamel.yaml.round_trip_dump(yaml_content, default_flow_style=False) kube_controller_manager['obj_count'] = str(len(crf_nodes)) check_kubernetes_object(kube_controller_manager, test_kubernetes_object_not_available, timeout=300) for key in crf_nodes: ex.execute_unix_command_on_remote_as_root("echo \"" + yaml_content + "\" > /etc/kubernetes/manifests/" + manifest_file, crf_nodes[key]) ex.execute_unix_command_on_remote_as_root("rm -f /tmp/" + manifest_file, crf_nodes[key]) check_kubernetes_object(kube_controller_manager, test_kubernetes_object_available, additional_filter="Running", timeout=300) wait_for_healthy_kube_controller_manager() @robot_log def add_flag_to_command(yaml_content, flag): yaml_content["spec"]["containers"][0]["command"].append(flag) @robot_log def remove_flag_from_command(yaml_content, flag): yaml_content["spec"]["containers"][0]["command"].remove(flag) @robot_log def helm_install(chart_name, release_name, values="registry_url={reg_url}".format(reg_url=reg)): command = "helm install " + chart_name + " --name " + release_name if values: command += " --set " + values ex.execute_unix_command(command, fail_on_non_zero_rc=False) if helm_list(release_name) == '1': logger.info(chart_name + " chart is successfully installed") else: raise Exception(chart_name + " chart install has failed.") @robot_log def helm_delete(release_name): ex.execute_unix_command("helm delete " + release_name + " --purge ", delay="30s", fail_on_non_zero_rc=False) if helm_list(release_name) == '0': logger.info(release_name + " chart is successfully deleted") else: raise Exception(release_name + " chart delete has failed.") @robot_log def helm_list(release_name, add_check_arg=''): grep_arg = 'grep -w {}'.format(release_name) if add_check_arg != '': grep_arg += '| grep -w {}'.format(add_check_arg) command = "helm list --all | {} | wc -l".format(grep_arg) stdout, _ = ex.execute_unix_command(command, fail_on_non_zero_rc=False) return stdout.strip() @robot_log def check_kubernetes_object(kube_object, tester_function, additional_filter=".*", timeout=0, delay=0): """" This method executes kubectl get command with the given args, filters the output and checks the result with the given tester_function. :param kube_object: a dictionary, it represents a kubernetes objects, obj_type, obj_name, namespace, obj_count keys are required. :param tester_function: this functoin checks the result and waits for the expected result - kubernetes object exists or not - to happen in a given time :param additional_filter: use this regexp to filter further the results :param timeout: wait seconds for the result :param delay: wait seconds before tester command """"" command = "kubectl get {object} -n {ns_arg} 2>/dev/null | grep -w {name} | grep -E '{grep_arg}' | wc -l" command = command.format(object=kube_object['obj_type'], name=kube_object['obj_name'], ns_arg=kube_object['namespace'], grep_arg=additional_filter) tester_function(kube_object, timeout, command, delay) @robot_log def is_result_expected_within_given_time(command, expected_result, timeout, delay=0): time.sleep(delay) result = ex.execute_unix_command(command) if result == expected_result: return True wait_until = datetime.now() + timedelta(seconds=timeout) while result != expected_result and (datetime.now() < wait_until): logger.info("datetime.now:" + str(datetime.now())) logger.info("wait_until:" + str(wait_until)) logger.info("expected result: " + expected_result) logger.info("result: " + result) time.sleep(1) result = ex.execute_unix_command(command) if result == expected_result: return True return False def test_kubernetes_object_quality(kube_object, expected_result, filter=".*", timeout=30, delay=0): tester_command = "kubectl get " + kube_object['obj_type'] + " --all-namespaces | grep -w " + \ kube_object['obj_name'] + " | grep -E '" + filter + "' | wc -l" res = is_result_expected_within_given_time(tester_command, expected_result, timeout, delay) if not res: log_command = "kubectl get " + kube_object['obj_type'] + " --all-namespaces | grep -w " + \ kube_object['obj_name'] res = ex.execute_unix_command(log_command) ex.execute_unix_command("kubectl describe " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " -n " + kube_object['namespace']) raise Exception("Not " + kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " is in expected (" + filter + ") state:" + res) logger.console(kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " is in expected (" + filter + ") state.") def test_kubernetes_object_available(kube_object, timeout, tester_command, delay=0): res = is_result_expected_within_given_time(tester_command, kube_object['obj_count'], timeout=timeout, delay=delay) if not res: describe_command = "kubectl describe " + kube_object['obj_type'] + " -n " + \ kube_object['namespace'] + " " + kube_object['obj_name'] ex.execute_unix_command(describe_command, fail_on_non_zero_rc=False) raise Exception("Not " + kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " is running!") logger.console(kube_object['obj_count'] + " " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " is running, as expected!") def test_kubernetes_object_not_available(kube_object, timeout, tester_command, delay=0): res = is_result_expected_within_given_time(tester_command, expected_result="0", timeout=timeout, delay=delay) if not res: describe_command = "kubectl describe " + kube_object['obj_type'] + " -n " + \ kube_object['namespace'] + " " + kube_object['obj_name'] ex.execute_unix_command(describe_command, fail_on_non_zero_rc=False) raise Exception("At least 1 " + kube_object['obj_type'] + " " + kube_object['obj_name'] + " still exists!") logger.console(kube_object['obj_type'] + " " + kube_object['obj_name'] + " does not exist, as expected!") def is_node_under_pressure(nodeslog): return bool(nodeslog.find("pressure") != -1) def wait_if_pressure(timeout=pressure_default_timeout): wait_until = datetime.now() + timedelta(seconds=timeout) command = "kubectl get nodes -o json | jq '.items[] | \"\(.metadata.name) \(.spec.taints)\"'" nodeslog = ex.execute_unix_command_as_root(command) while (is_node_under_pressure(nodeslog)) and (datetime.now() < wait_until): logger.info("datetime.now:" + str(datetime.now())) logger.info("wait_until:" + str(wait_until)) logger.info("Node is under pressure found: " + nodeslog) time.sleep(10) nodeslog = ex.execute_unix_command_as_root(command) if is_node_under_pressure(nodeslog): raise Exception("Node pressure not resolved in time.") else: logger.info(nodeslog) @robot_log def check_url_running(filename, url): command = "curl -s {url} > /dev/null ; echo -n $?" result = ex.execute_unix_command_as_root(command.format(url=url)) if result == "0": logger.console("{url} is running!".format(url=url)) else: gather_logs("curl -s {url}".format(url=url), filename, LOG_DIR) raise Exception("{url} is not running !".format(url=url)) @robot_log def subprocess_cmd(command): return subprocess.check_output(command, shell=True).strip() @robot_log def put_file(local_script_path, remote_script_path, permissions="777", user=root['username'], group=root['username']): ex.get_ssh_library_instance().put_file(local_script_path, remote_script_path, permissions) head, tail = os.path.split(remote_script_path) command = 'ls -l ' + head + ' | grep ' + tail + ' | wc -l' res = is_result_expected_within_given_time(command, expected_result="1", timeout=5) if not res: raise Exception("File not found at " + remote_script_path + "!") ex.execute_unix_command_as_root('chgrp ' + group + ' ' + remote_script_path) ex.execute_unix_command_as_root('chown ' + user + ' ' + remote_script_path) @robot_log def get_helm_chart_content(chart_name): ex.execute_unix_command("helm fetch " + chart_name + " --untar --untardir /tmp") return ex.execute_unix_command("ls /tmp/" + chart_name.split('/')[1] + "/templates | awk -F . '{print $1}'").split('\r\n') @robot_log def get_cpupools(): node_map = {} node_list = ex.execute_unix_command("kubectl get nodes -L=nodename | awk '{print $6}'| tail -n +2") cmap_str = ex.execute_unix_command("kubectl get configmap -n kube-system {cm} -o yaml" .format(cm=cpu_pooling_cm_name)) for nodename in node_list.splitlines(): # pylint: disable=too-many-nested-blocks yamldict = yaml.load(cmap_str) for key in yamldict['data']: if nodename in yamldict['data'][key]: worker_yaml = yaml.load(yamldict['data'][key]) pool_dict = {} if worker_yaml['pools']: for pool in worker_yaml['pools']: pool_str = worker_yaml['pools'][pool]['cpus'] pool_list = [] for sub_list in pool_str.split(','): pool_list = pool_list + ([int(sub_list)] if '-' not in sub_list else range(int(sub_list.split('-')[0]), int(sub_list.split('-')[1]) + 1)) pool_dict[pool] = pool_list node_map[nodename] = pool_dict return node_map @robot_log def get_cpu_allowed_list_from_pod(pod_name): bash_command = "cat /proc/1/status | grep Cpus_allowed_list" result = ex.execute_unix_command("kubectl exec `kubectl get pod | grep {0} | " "awk '{{print $1}}'` -- {1}".format(pod_name, bash_command)) pool_list = [] for cpu in result.split(':')[1].split(','): pool_list = pool_list + ([int(cpu)] if '-' not in cpu else range(int(cpu.split('-')[0]), int(cpu.split('-')[1]) + 1)) return pool_list @robot_log def allowed_cpus_is_in_cpu_pool(allowed_cpus, cpu_pool): for allowed in allowed_cpus: if allowed not in cpu_pool: return False return True def decide_nodename(): nodename = 'caas_worker1' command = "kubectl get node -L=nodename | awk {{'print $6'}} | tail -n +2" node_names = ex.execute_unix_command(command) if nodename not in node_names: return node_names.splitlines()[0] return nodename @robot_log def determine_accurate_running_time_of_obj(object_type, object_name): hours = mins = secs = 0 cmd = "kubectl get {obj_type} --all-namespaces --no-headers=true | grep {obj_name} | awk '{{print $NF}}'" \ .format(obj_type=object_type, obj_name=object_name) resp = ex.execute_unix_command(cmd) pod_time = re.findall(r'\d{0,2}h|\d{0,3}m|\d{1,3}s', resp) for t in pod_time: if t[-1] == 'h': hours = int(t[:-1]) elif t[-1] == 'm': mins = int(t[:-1]) elif t[-1] == 's': secs = int(t[:-1]) return datetime.now() - timedelta(hours=hours, minutes=mins, seconds=secs)