X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fcloudtaf.git;a=blobdiff_plain;f=testcases%2Fdanm_network_check%2Fdanm_utils.py;fp=testcases%2Fdanm_network_check%2Fdanm_utils.py;h=2cf801a47c0f54a265e8d4a0499a650d28120302;hp=0000000000000000000000000000000000000000;hb=af5eb3ff36b92ab1d9c156ffa0391eadc73eb6ba;hpb=025a45508d009db84c34076fb4a668f712628d6d diff --git a/testcases/danm_network_check/danm_utils.py b/testcases/danm_network_check/danm_utils.py new file mode 100644 index 0000000..2cf801a --- /dev/null +++ b/testcases/danm_network_check/danm_utils.py @@ -0,0 +1,238 @@ +import time +import json +import re +import os +import users + +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn +from netaddr import IPAddress +from decorators_for_robot_functionalities import * + + +log_dir = os.path.join(os.path.dirname(__file__)) +ex = BuiltIn().get_library_instance('execute_command') +sshlib = ex.get_ssh_library_instance() +stack_infos = BuiltIn().get_library_instance('stack_infos') +BuiltIn().import_library('pabot.PabotLib') +pabot = BuiltIn().get_library_instance('pabot.PabotLib') + + +@robot_log +def check_host_interfaces(network_properties_dict): + for node, node_ip in stack_infos.get_all_nodes().items(): + logger.info("Checking host interfaces on " + node + ": " + node_ip) + ex.ssh_to_another_node(node_ip, users.cloudadmin) + try: + for network in network_properties_dict: + if network_properties_dict[network]['host_if'] != '': + if (node in stack_infos.get_worker_nodes() and + network_properties_dict[network]['iface_type'] != 'int'): + continue + command = "ip a | grep " + network_properties_dict[network]['host_if'] + " | wc -l" + count = ex.execute_unix_command(command) + if count != '1': + raise Exception("host interface check Failed, interface " + + network_properties_dict[network]['host_if'] + + " does not exist on " + node + ": " + node_ip) + logger.info("host interface check OK, interface " + network_properties_dict[network]['host_if'] + + " exists on " + node) + else: + command = "ip a | grep " + network_properties_dict[network]['name'] + "[.:] | wc -l" + count = ex.execute_unix_command(command) + if count != '0': + raise Exception("host interface check Failed, " + network_properties_dict[network]['name'] + + " related interface exists on node: " + node + ": " + node_ip) + logger.info("host interface check OK, no unnecessary " + network_properties_dict[network]['name'] + + " related host interface exists on node: " + node + ": " + node_ip) + finally: + ex.exit_from_user() + + +@robot_log +def create_resources_from_fetched_chart_templates(template_path): + ex.execute_unix_command("kubectl create -f " + template_path, fail_on_non_zero_rc=False) + + +@robot_log +def delete_all_resources(resource_type): + ex.execute_unix_command("kubectl delete " + resource_type + " --all") + + +@robot_log +def delete_resources_by_manifest_path(path): + ex.execute_unix_command("kubectl delete -f " + path) + + +@robot_log +def get_resource_count(resource_type, resource_name): + return ex.execute_unix_command("kubectl get " + resource_type + " 2>/dev/null | grep -w " + resource_name + + " | wc -l") + + +@robot_log +def compare_test_data(list_to_compare, dict_to_compare): + for danmnet in list_to_compare: + if danmnet not in dict_to_compare: + logger.warn(danmnet + " is not present in test constants: {}".format(dict_to_compare)) + for key in dict_to_compare: + if key not in list_to_compare: + logger.warn(key + " is not present in {} chart".format(list_to_compare)) + + +@robot_log +def get_pod_list(kube_object): + pod_list = {} + command = "kubectl get pod --all-namespaces | grep -w " + kube_object[ + 'obj_name'] + " | awk '{print $1 \" \" $2 \" \" $4 \" \" $5}'" + for line in ex.execute_unix_command_as_root(command).split('\r\n'): + pod_list[line.split(' ')[1]] = {'namespace': line.split(' ')[0], 'status': line.split(' ')[2], + 'restarts': line.split(' ')[3]} + return pod_list + + +@robot_log +def get_pod_ips(pod_list, skip_restarts=False, if_name='eth0'): + assigned_ips = [] + for key in pod_list: + if (pod_list[key]['status'] == 'Running') and ((pod_list[key]['restarts'] == '0') or skip_restarts): + logger.info(pod_list[key]['namespace']) + if if_name != '': + command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + " ip a | grep " + if_name + \ + " | grep inet | awk '{print $2}' | awk -F \"/\" '{print $1}' " + else: + command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + " -- ip -o a | " \ + "grep -vE '(: lo|: eth0)' | grep inet | awk '{print $4}' | awk -F \"/\" '{print $1}'" + assigned_ips.append(ex.execute_unix_command_as_root(command)) + return assigned_ips + + +@robot_log +def check_mac_address(pod_list, network, prop_dict): + command = "ip a | grep -wA 1 " + prop_dict[network]['host_if'] + " | grep ether | awk '{print $2}'" + host_mac = ex.execute_unix_command_as_root(command) + for pod in pod_list: + if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'): + command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " ip a | grep -A 1 eth0 | " \ + "grep link | awk '{print $2}'" + pod_mac = ex.execute_unix_command_as_root(command) + if host_mac != pod_mac: + raise Exception("Wrong Mac address in pod " + pod + "hostmac: " + host_mac + " ; podmac: " + pod_mac) + logger.info("Correct mac address in pod " + pod) + + +@robot_log +def get_alloc_pool(network, dictionary, resource_type): + alloc_pool = {} + command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -n " + \ + dictionary[network]['namespace'] + " -o yaml " + \ + " | grep allocation_pool -A 2 | grep start | awk {'print$2'}" + alloc_pool['start'] = ex.execute_unix_command_as_root(command) + command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -n " + \ + dictionary[network]['namespace'] + " -o yaml " + \ + " | grep allocation_pool -A 2 | grep end | awk {'print$2'}" + alloc_pool['end'] = ex.execute_unix_command_as_root(command) + return alloc_pool + + +@robot_log +def check_dynamic_ips(alloc_pool, assigned_ips): + for ip in assigned_ips: + if (IPAddress(alloc_pool['start']) > IPAddress(ip)) or (IPAddress(ip) > IPAddress(alloc_pool['end'])): + raise Exception("Dynamic ip: {} is not in allocation pool: {} - {}".format(ip, alloc_pool['start'], + alloc_pool['end'])) + logger.info("All dynamic ips are from the allocation pool.") + if len((set(assigned_ips))) != len(assigned_ips): + raise Exception("duplicated IPs assigned") + logger.info("All allocated IPs are unique") + + +@robot_log +def check_static_routes(pod_list, network, properties_dict): + for pod in pod_list: + if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'): + command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " route | grep " + \ + properties_dict[network]['routes'].split('/')[0] + " | grep " + \ + properties_dict[network]['routes'].split(' ')[1] + " | wc -l" + res = ex.execute_unix_command_as_root(command) + if res != '1': + raise Exception("static route in pod " + pod + " does not match with route defined in " + network) + logger.info("Static route in pod " + pod + " is as it should be.") + + +@robot_log +def check_connectivity(pod_list, pod, ip_list): + for ip in ip_list: + command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " -- sh -c \"ping -c 1 " + ip + "\"" + stdout = ex.execute_unix_command_as_root(command) + if '0% packet loss' not in stdout: + raise Exception("pod " + pod + " cannot reach ip " + ip) + logger.info("pod " + pod + " can reach ip " + ip) + + +@robot_log +def check_danmnet_endpoints_deleted(kube_object, network, properties_dict, assigned_ips): + for ip in assigned_ips: + command = "kubectl get danmep -n " + kube_object['namespace'] + " -o yaml | grep -B 10 " + \ + properties_dict[network]['name'] + " | grep " + ip + " | wc -l" + res = ex.execute_unix_command_as_root(command) + if res != '0': + raise Exception("Endpoint with ip " + ip + " still exists.") + logger.info("The necessary endpoints are cleared") + + +@robot_log +def get_alloc_value(network, dictionary, resource_type): + command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -o yaml | grep -w alloc | " \ + "awk '{print $2}'" + alloc = ex.execute_unix_command_as_root(command) + return alloc + + +def check_danm_count(ip_count_before_parameter, cbr0_content1_parameter, tries): + if tries == 5: + raise Exception("Flannel ips are not cleared after pod deletion") + else: + tries = tries + 1 + command = "ls -lrt /var/lib/cni/networks/cbr0/ | wc -l" + ip_count_after = ex.execute_unix_command_as_root(command) + command = "ls -lrt /var/lib/cni/networks/cbr0/" + cbr0_content2 = ex.execute_unix_command_as_root(command) + ip_count_before = ip_count_before_parameter + cbr0_content1 = cbr0_content1_parameter + if ip_count_before != ip_count_after: + logger.info(cbr0_content1) + logger.info(cbr0_content2) + time.sleep(30) + check_danm_count(ip_count_before, cbr0_content1, tries) + + +@robot_log +def check_dep_count(namespace, exp_count, test_pod_name_pattern=r'^danmnet-pods'): + tries = 0 + deps = get_deps(namespace) + # test_pod_name_pattern = r'^danmnet-pods' + danmnet_test_deps = [dep for dep in deps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)] + while (tries < 5) and (len(danmnet_test_deps) != exp_count): + time.sleep(20) + tries += 1 + deps = get_deps(namespace) + danmnet_test_deps = [dep for dep in deps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)] + + if len(danmnet_test_deps) != exp_count: + raise Exception("Danm endpoint count is not as expected! Got: " + str(len(danmnet_test_deps)) + ", expected: " + + str(exp_count)) + logger.info("Danm endpoint count is as expected.") + + +@robot_log +def get_deps(namespace): + command = "kubectl get dep -n {} -o json".format(namespace) + deps_text = ex.execute_unix_command_as_root(command) + return json.loads(deps_text).get("items") + + +@robot_log +def is_dep_belongs_to_pod(dep, pod_pattern): + pod_name = dep["spec"]["Pod"] + return bool(re.search(pod_pattern, pod_name))