testcases/danm_network_check/danm_utils.py

   1 import time
   2 import json
   3 import re
   4 import os
   5 import users
   6
   7 from robot.api import logger
   8 from robot.libraries.BuiltIn import BuiltIn
   9 from netaddr import IPAddress
  10 from decorators_for_robot_functionalities import *
  11
  12
  13 log_dir = os.path.join(os.path.dirname(__file__))
  14 ex = BuiltIn().get_library_instance('execute_command')
  15 sshlib = ex.get_ssh_library_instance()
  16 stack_infos = BuiltIn().get_library_instance('stack_infos')
  17 BuiltIn().import_library('pabot.PabotLib')
  18 pabot = BuiltIn().get_library_instance('pabot.PabotLib')
  19
  20
  21 @robot_log
  22 def check_host_interfaces(network_properties_dict):
  23     for node, node_ip in stack_infos.get_all_nodes().items():
  24         logger.info("Checking host interfaces on " + node + ": " + node_ip)
  25         ex.ssh_to_another_node(node_ip, users.cloudadmin)
  26         try:
  27             for network in network_properties_dict:
  28                 if network_properties_dict[network]['host_if'] != '':
  29                     if (node in stack_infos.get_worker_nodes() and
  30                             network_properties_dict[network]['iface_type'] != 'int'):
  31                         continue
  32                     command = "ip a | grep " + network_properties_dict[network]['host_if'] + " | wc -l"
  33                     count = ex.execute_unix_command(command)
  34                     if count != '1':
  35                         raise Exception("host interface check Failed, interface " +
  36                                         network_properties_dict[network]['host_if'] +
  37                                         " does not exist on " + node + ": " + node_ip)
  38                     logger.info("host interface check OK, interface " + network_properties_dict[network]['host_if'] +
  39                                 " exists on " + node)
  40                 else:
  41                     command = "ip a | grep " + network_properties_dict[network]['name'] + "[.:] | wc -l"
  42                     count = ex.execute_unix_command(command)
  43                     if count != '0':
  44                         raise Exception("host interface check Failed, " + network_properties_dict[network]['name'] +
  45                                         " related interface exists on node: " + node + ": " + node_ip)
  46                     logger.info("host interface check OK, no unnecessary " + network_properties_dict[network]['name'] +
  47                                 " related host interface exists on node: " + node + ": " + node_ip)
  48         finally:
  49             ex.exit_from_user()
  50
  51
  52 @robot_log
  53 def create_resources_from_fetched_chart_templates(template_path):
  54     ex.execute_unix_command("kubectl create -f " + template_path, fail_on_non_zero_rc=False)
  55
  56
  57 @robot_log
  58 def delete_all_resources(resource_type):
  59     ex.execute_unix_command("kubectl delete " + resource_type + " --all")
  60
  61
  62 @robot_log
  63 def delete_resources_by_manifest_path(path):
  64     ex.execute_unix_command("kubectl delete -f " + path)
  65
  66
  67 @robot_log
  68 def get_resource_count(resource_type, resource_name):
  69     return ex.execute_unix_command("kubectl get " + resource_type + " 2>/dev/null | grep -w " + resource_name +
  70                                    " | wc -l")
  71
  72
  73 @robot_log
  74 def compare_test_data(list_to_compare, dict_to_compare):
  75     for danmnet in list_to_compare:
  76         if danmnet not in dict_to_compare:
  77             logger.warn(danmnet + " is not present in test constants: {}".format(dict_to_compare))
  78     for key in dict_to_compare:
  79         if key not in list_to_compare:
  80             logger.warn(key + " is not present in {} chart".format(list_to_compare))
  81
  82
  83 @robot_log
  84 def get_pod_list(kube_object):
  85     pod_list = {}
  86     command = "kubectl get pod --all-namespaces | grep -w " + kube_object[
  87         'obj_name'] + " | awk '{print $1 \" \" $2 \" \" $4 \" \" $5}'"
  88     for line in ex.execute_unix_command_as_root(command).split('\r\n'):
  89         pod_list[line.split(' ')[1]] = {'namespace': line.split(' ')[0], 'status': line.split(' ')[2],
  90                                         'restarts': line.split(' ')[3]}
  91     return pod_list
  92
  93
  94 @robot_log
  95 def get_pod_ips(pod_list, skip_restarts=False, if_name='eth0'):
  96     assigned_ips = []
  97     for key in pod_list:
  98         if (pod_list[key]['status'] == 'Running') and ((pod_list[key]['restarts'] == '0') or skip_restarts):
  99             logger.info(pod_list[key]['namespace'])
 100             if if_name != '':
 101                 command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + " ip a | grep " + if_name + \
 102                           " | grep inet | awk '{print $2}' | awk -F \"/\" '{print $1}' "
 103             else:
 104                 command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + "  -- ip -o a | " \
 105                           "grep -vE '(: lo|: eth0)' | grep inet | awk '{print $4}' | awk -F \"/\" '{print $1}'"
 106             assigned_ips.append(ex.execute_unix_command_as_root(command))
 107     return assigned_ips
 108
 109
 110 @robot_log
 111 def check_mac_address(pod_list, network, prop_dict):
 112     command = "ip a | grep -wA 1 " + prop_dict[network]['host_if'] + " | grep ether | awk '{print $2}'"
 113     host_mac = ex.execute_unix_command_as_root(command)
 114     for pod in pod_list:
 115         if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'):
 116             command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " ip a | grep -A 1 eth0 | " \
 117                                                                                     "grep link | awk '{print $2}'"
 118             pod_mac = ex.execute_unix_command_as_root(command)
 119             if host_mac != pod_mac:
 120                 raise Exception("Wrong Mac address in pod " + pod + "hostmac: " + host_mac + " ; podmac: " + pod_mac)
 121             logger.info("Correct mac address in pod " + pod)
 122
 123
 124 @robot_log
 125 def get_alloc_pool(network, dictionary, resource_type):
 126     alloc_pool = {}
 127     command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -n " + \
 128               dictionary[network]['namespace'] + " -o yaml " + \
 129               " | grep allocation_pool -A 2 | grep start | awk {'print$2'}"
 130     alloc_pool['start'] = ex.execute_unix_command_as_root(command)
 131     command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -n " + \
 132               dictionary[network]['namespace'] + " -o yaml " + \
 133               " | grep allocation_pool -A 2 | grep end | awk {'print$2'}"
 134     alloc_pool['end'] = ex.execute_unix_command_as_root(command)
 135     return alloc_pool
 136
 137
 138 @robot_log
 139 def check_dynamic_ips(alloc_pool, assigned_ips):
 140     for ip in assigned_ips:
 141         if (IPAddress(alloc_pool['start']) > IPAddress(ip)) or (IPAddress(ip) > IPAddress(alloc_pool['end'])):
 142             raise Exception("Dynamic ip: {} is not in allocation pool: {} - {}".format(ip, alloc_pool['start'],
 143                                                                                        alloc_pool['end']))
 144     logger.info("All dynamic ips are from the allocation pool.")
 145     if len((set(assigned_ips))) != len(assigned_ips):
 146         raise Exception("duplicated IPs assigned")
 147     logger.info("All allocated IPs are unique")
 148
 149
 150 @robot_log
 151 def check_static_routes(pod_list, network, properties_dict):
 152     for pod in pod_list:
 153         if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'):
 154             command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " route | grep " + \
 155                       properties_dict[network]['routes'].split('/')[0] + " | grep " + \
 156                       properties_dict[network]['routes'].split(' ')[1] + " | wc -l"
 157             res = ex.execute_unix_command_as_root(command)
 158             if res != '1':
 159                 raise Exception("static route in pod " + pod + " does not match with route defined in " + network)
 160             logger.info("Static route in pod " + pod + " is as it should be.")
 161
 162
 163 @robot_log
 164 def check_connectivity(pod_list, pod, ip_list):
 165     for ip in ip_list:
 166         command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " -- sh -c \"ping -c 1 " + ip + "\""
 167         stdout = ex.execute_unix_command_as_root(command)
 168         if '0% packet loss' not in stdout:
 169             raise Exception("pod " + pod + " cannot reach ip " + ip)
 170         logger.info("pod " + pod + " can reach ip " + ip)
 171
 172
 173 @robot_log
 174 def check_danmnet_endpoints_deleted(kube_object, network, properties_dict, assigned_ips):
 175     for ip in assigned_ips:
 176         command = "kubectl get danmep -n " + kube_object['namespace'] + " -o yaml | grep -B 10 " + \
 177                   properties_dict[network]['name'] + " | grep " + ip + " | wc -l"
 178         res = ex.execute_unix_command_as_root(command)
 179         if res != '0':
 180             raise Exception("Endpoint with ip " + ip + " still exists.")
 181     logger.info("The necessary endpoints are cleared")
 182
 183
 184 @robot_log
 185 def get_alloc_value(network, dictionary, resource_type):
 186     command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -o yaml | grep -w alloc | " \
 187                                                                                    "awk '{print $2}'"
 188     alloc = ex.execute_unix_command_as_root(command)
 189     return alloc
 190
 191
 192 def check_danm_count(ip_count_before_parameter, cbr0_content1_parameter, tries):
 193     if tries == 5:
 194         raise Exception("Flannel ips are not cleared after pod deletion")
 195     else:
 196         tries = tries + 1
 197     command = "ls -lrt /var/lib/cni/networks/cbr0/ | wc -l"
 198     ip_count_after = ex.execute_unix_command_as_root(command)
 199     command = "ls -lrt /var/lib/cni/networks/cbr0/"
 200     cbr0_content2 = ex.execute_unix_command_as_root(command)
 201     ip_count_before = ip_count_before_parameter
 202     cbr0_content1 = cbr0_content1_parameter
 203     if ip_count_before != ip_count_after:
 204         logger.info(cbr0_content1)
 205         logger.info(cbr0_content2)
 206         time.sleep(30)
 207         check_danm_count(ip_count_before, cbr0_content1, tries)
 208
 209
 210 @robot_log
 211 def check_dep_count(namespace, exp_count, test_pod_name_pattern=r'^danmnet-pods'):
 212     tries = 0
 213     deps = get_deps(namespace)
 214     # test_pod_name_pattern = r'^danmnet-pods'
 215     danmnet_test_deps = [dep for dep in deps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)]
 216     while (tries < 5) and (len(danmnet_test_deps) != exp_count):
 217         time.sleep(20)
 218         tries += 1
 219         deps = get_deps(namespace)
 220         danmnet_test_deps = [dep for dep in deps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)]
 221
 222     if len(danmnet_test_deps) != exp_count:
 223         raise Exception("Danm endpoint count is not as expected! Got: " + str(len(danmnet_test_deps)) + ", expected: " +
 224                         str(exp_count))
 225     logger.info("Danm endpoint count is as expected.")
 226
 227
 228 @robot_log
 229 def get_deps(namespace):
 230     command = "kubectl get dep -n {} -o json".format(namespace)
 231     deps_text = ex.execute_unix_command_as_root(command)
 232     return json.loads(deps_text).get("items")
 233
 234
 235 @robot_log
 236 def is_dep_belongs_to_pod(dep, pod_pattern):
 237     pod_name = dep["spec"]["Pod"]
 238     return bool(re.search(pod_pattern, pod_name))