7 from robot.api import logger
8 from robot.libraries.BuiltIn import BuiltIn
9 from netaddr import IPAddress
10 from decorators_for_robot_functionalities import *
13 log_dir = os.path.join(os.path.dirname(__file__))
14 ex = BuiltIn().get_library_instance('execute_command')
15 sshlib = ex.get_ssh_library_instance()
16 stack_infos = BuiltIn().get_library_instance('stack_infos')
17 BuiltIn().import_library('pabot.PabotLib')
18 pabot = BuiltIn().get_library_instance('pabot.PabotLib')
22 def check_host_interfaces(network_properties_dict):
23 for node, node_ip in stack_infos.get_all_nodes().items():
24 logger.info("Checking host interfaces on " + node + ": " + node_ip)
25 ex.ssh_to_another_node(node_ip, users.cloudadmin)
27 for network in network_properties_dict:
28 if network_properties_dict[network]['host_if'] != '':
29 if (node in stack_infos.get_worker_nodes() and
30 network_properties_dict[network]['iface_type'] != 'int'):
32 command = "ip a | grep " + network_properties_dict[network]['host_if'] + " | wc -l"
33 count = ex.execute_unix_command(command)
35 raise Exception("host interface check Failed, interface " +
36 network_properties_dict[network]['host_if'] +
37 " does not exist on " + node + ": " + node_ip)
38 logger.info("host interface check OK, interface " + network_properties_dict[network]['host_if'] +
41 command = "ip a | grep " + network_properties_dict[network]['name'] + "[.:] | wc -l"
42 count = ex.execute_unix_command(command)
44 raise Exception("host interface check Failed, " + network_properties_dict[network]['name'] +
45 " related interface exists on node: " + node + ": " + node_ip)
46 logger.info("host interface check OK, no unnecessary " + network_properties_dict[network]['name'] +
47 " related host interface exists on node: " + node + ": " + node_ip)
53 def create_resources_from_fetched_chart_templates(template_path):
54 ex.execute_unix_command("kubectl create -f " + template_path, fail_on_non_zero_rc=False)
58 def delete_all_resources(resource_type):
59 ex.execute_unix_command("kubectl delete " + resource_type + " --all")
63 def delete_resources_by_manifest_path(path):
64 ex.execute_unix_command("kubectl delete -f " + path)
68 def get_resource_count(resource_type, resource_name):
69 return ex.execute_unix_command("kubectl get " + resource_type + " 2>/dev/null | grep -w " + resource_name +
74 def compare_test_data(list_to_compare, dict_to_compare):
75 for danmnet in list_to_compare:
76 if danmnet not in dict_to_compare:
77 logger.warn(danmnet + " is not present in test constants: {}".format(dict_to_compare))
78 for key in dict_to_compare:
79 if key not in list_to_compare:
80 logger.warn(key + " is not present in {} chart".format(list_to_compare))
84 def get_pod_list(kube_object):
86 command = "kubectl get pod --all-namespaces | grep -w " + kube_object[
87 'obj_name'] + " | awk '{print $1 \" \" $2 \" \" $4 \" \" $5}'"
88 for line in ex.execute_unix_command_as_root(command).split('\r\n'):
89 pod_list[line.split(' ')[1]] = {'namespace': line.split(' ')[0], 'status': line.split(' ')[2],
90 'restarts': line.split(' ')[3]}
95 def get_pod_ips(pod_list, skip_restarts=False, if_name='eth0'):
98 if (pod_list[key]['status'] == 'Running') and ((pod_list[key]['restarts'] == '0') or skip_restarts):
99 logger.info(pod_list[key]['namespace'])
101 command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + " ip a | grep " + if_name + \
102 " | grep inet | awk '{print $2}' | awk -F \"/\" '{print $1}' "
104 command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + " -- ip -o a | " \
105 "grep -vE '(: lo|: eth0)' | grep inet | awk '{print $4}' | awk -F \"/\" '{print $1}'"
106 assigned_ips.append(ex.execute_unix_command_as_root(command))
111 def check_mac_address(pod_list, network, prop_dict):
112 command = "ip a | grep -wA 1 " + prop_dict[network]['host_if'] + " | grep ether | awk '{print $2}'"
113 host_mac = ex.execute_unix_command_as_root(command)
115 if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'):
116 command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " ip a | grep -A 1 eth0 | " \
117 "grep link | awk '{print $2}'"
118 pod_mac = ex.execute_unix_command_as_root(command)
119 if host_mac != pod_mac:
120 raise Exception("Wrong Mac address in pod " + pod + "hostmac: " + host_mac + " ; podmac: " + pod_mac)
121 logger.info("Correct mac address in pod " + pod)
125 def get_alloc_pool(network, dictionary, resource_type):
127 command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -n " + \
128 dictionary[network]['namespace'] + " -o yaml " + \
129 " | grep allocation_pool -A 2 | grep start | awk {'print$2'}"
130 alloc_pool['start'] = ex.execute_unix_command_as_root(command)
131 command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -n " + \
132 dictionary[network]['namespace'] + " -o yaml " + \
133 " | grep allocation_pool -A 2 | grep end | awk {'print$2'}"
134 alloc_pool['end'] = ex.execute_unix_command_as_root(command)
139 def check_dynamic_ips(alloc_pool, assigned_ips):
140 for ip in assigned_ips:
141 if (IPAddress(alloc_pool['start']) > IPAddress(ip)) or (IPAddress(ip) > IPAddress(alloc_pool['end'])):
142 raise Exception("Dynamic ip: {} is not in allocation pool: {} - {}".format(ip, alloc_pool['start'],
144 logger.info("All dynamic ips are from the allocation pool.")
145 if len((set(assigned_ips))) != len(assigned_ips):
146 raise Exception("duplicated IPs assigned")
147 logger.info("All allocated IPs are unique")
151 def check_static_routes(pod_list, network, properties_dict):
153 if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'):
154 command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " route | grep " + \
155 properties_dict[network]['routes'].split('/')[0] + " | grep " + \
156 properties_dict[network]['routes'].split(' ')[1] + " | wc -l"
157 res = ex.execute_unix_command_as_root(command)
159 raise Exception("static route in pod " + pod + " does not match with route defined in " + network)
160 logger.info("Static route in pod " + pod + " is as it should be.")
164 def check_connectivity(pod_list, pod, ip_list):
166 command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " -- sh -c \"ping -c 1 " + ip + "\""
167 stdout = ex.execute_unix_command_as_root(command)
168 if '0% packet loss' not in stdout:
169 raise Exception("pod " + pod + " cannot reach ip " + ip)
170 logger.info("pod " + pod + " can reach ip " + ip)
174 def check_danmnet_endpoints_deleted(kube_object, network, properties_dict, assigned_ips):
175 for ip in assigned_ips:
176 command = "kubectl get danmep -n " + kube_object['namespace'] + " -o yaml | grep -B 10 " + \
177 properties_dict[network]['name'] + " | grep " + ip + " | wc -l"
178 res = ex.execute_unix_command_as_root(command)
180 raise Exception("Endpoint with ip " + ip + " still exists.")
181 logger.info("The necessary endpoints are cleared")
185 def get_alloc_value(network, dictionary, resource_type):
186 command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -o yaml | grep -w alloc | " \
188 alloc = ex.execute_unix_command_as_root(command)
192 def check_danm_count(ip_count_before_parameter, cbr0_content1_parameter, tries):
194 raise Exception("Flannel ips are not cleared after pod deletion")
197 command = "ls -lrt /var/lib/cni/networks/cbr0/ | wc -l"
198 ip_count_after = ex.execute_unix_command_as_root(command)
199 command = "ls -lrt /var/lib/cni/networks/cbr0/"
200 cbr0_content2 = ex.execute_unix_command_as_root(command)
201 ip_count_before = ip_count_before_parameter
202 cbr0_content1 = cbr0_content1_parameter
203 if ip_count_before != ip_count_after:
204 logger.info(cbr0_content1)
205 logger.info(cbr0_content2)
207 check_danm_count(ip_count_before, cbr0_content1, tries)
211 def check_dep_count(namespace, exp_count, test_pod_name_pattern=r'^danmnet-pods'):
213 deps = get_deps(namespace)
214 # test_pod_name_pattern = r'^danmnet-pods'
215 danmnet_test_deps = [dep for dep in deps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)]
216 while (tries < 5) and (len(danmnet_test_deps) != exp_count):
219 deps = get_deps(namespace)
220 danmnet_test_deps = [dep for dep in deps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)]
222 if len(danmnet_test_deps) != exp_count:
223 raise Exception("Danm endpoint count is not as expected! Got: " + str(len(danmnet_test_deps)) + ", expected: " +
225 logger.info("Danm endpoint count is as expected.")
229 def get_deps(namespace):
230 command = "kubectl get dep -n {} -o json".format(namespace)
231 deps_text = ex.execute_unix_command_as_root(command)
232 return json.loads(deps_text).get("items")
236 def is_dep_belongs_to_pod(dep, pod_pattern):
237 pod_name = dep["spec"]["Pod"]
238 return bool(re.search(pod_pattern, pod_name))