testcases/danm_network_check/tc_003_clusternetwork_pod_check.py

   1 import sys\r
   2 import os\r
   3 import time\r
   4 import json\r
   5 import re\r
   6 import danm_utils\r
   7 from robot.api import logger\r
   8 from robot.libraries.BuiltIn import BuiltIn\r
   9 from netaddr import IPAddress\r
  10 \r
  11 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '../libraries/common'))\r
  12 import common_utils  # noqa\r
  13 from execute_command import execute_command  # noqa\r
  14 from decorators_for_robot_functionalities import *  # noqa\r
  15 from test_constants import *  # noqa\r
  16 \r
  17 \r
  18 execute = BuiltIn().get_library_instance('execute_command')\r
  19 stack_infos = BuiltIn().get_library_instance('stack_infos')\r
  20 flannel_pool = {'start': '10.244.0.1', 'end': '10.244.255.254'}\r
  21 static_ips = ["10.5.1.11", "10.5.1.19", "10.5.1.20", "10.5.255.254", ]\r
  22 infra_int_if = stack_infos.get_infra_int_if()\r
  23 infra_ext_if = stack_infos.get_infra_ext_if()\r
  24 infra_storage_if = stack_infos.get_infra_storage_if()\r
  25 \r
  26 \r
  27 def tc_003_clusternetwork_pod_check():\r
  28     """\r
  29         danmnet_pods1: pods attached to d_test-net2 with static ips\r
  30         danmnet_pods2: pods attached to d_test-net2 with dynamic ips\r
  31         danmnet_pods3: pods attached to d_test-net2 with the same static ips as danmnet-pods1\r
  32         danmnet_pods4: pods attached to ks_test-net2 with dynamic ips (kube-system namespace)\r
  33         danmnet_pods5: pod attached to d_test-net2 with static ip, ip is not in CIDR\r
  34         danmnet_pods6: pods attached to d_test-net1 with dynamic ips (no CIDR/alloc pool is defined in test-net1 )\r
  35         danmnet_pods7: pods attached to d_test-net24(flannel) and d_test-net7(ipvlan) networks with dynamic ip\r
  36         danmnet_pods8: pods attached to d_test-net24(flannel) with dynamic ip and service defined\r
  37         danmnet_pods9: pods attached to d_test-net24(flannel) with static ip(ignored)\r
  38         danmnet_pods10: pods attached to d_test-net2 with none ip\r
  39         danmnet_pods11: pod attached to d_test-net30 with static ip, d_test-net8 with none ip, none existing\r
  40         danmnet(error)\r
  41         danmnet_pods12: pod attached to d_test-net30 with static ip, d_test-net8 with dynamic ip, d_test-net25 with\r
  42         none ip\r
  43         danmnet_pods13: pod attached to d_test-net8 with static ip, d_test-net24(flannel) with dynamic ip, none existing\r
  44         danmnet(error)\r
  45         danmnet_pods14: pod attached to d_test-net25 with static ip, d_test-net24(flannel) with dynamic ip\r
  46 \r
  47         danmnet_pods1: pods attached to cnet-pod1 with static ips\r
  48         danmnet_pods2: pods attached to cnet-pod1 with dynamic ips\r
  49         danmnet_pods3: pods attached to cnet-pod1 with the same static ips as danmnet-pods1\r
  50         danmnet_pods4: pods attached to ks_test-net2 with dynamic ips (kube-system namespace)\r
  51         danmnet_pods5: pod attached to cnet-pod1 with static ip, ip is not in CIDR\r
  52         danmnet_pods6: pods attached to d_test-net1 with dynamic ips (no CIDR/alloc pool is defined in test-net1 )\r
  53         danmnet_pods7: pods attached to d_test-net24(flannel) and d_test-net7(ipvlan) networks with dynamic ip\r
  54         danmnet_pods8: pods attached to d_test-net24(flannel) with dynamic ip and service defined\r
  55         danmnet_pods9: pods attached to d_test-net24(flannel) with static ip(ignored)\r
  56         danmnet_pods10: pods attached to cnet-pod1 with none ip\r
  57         danmnet_pods11: pod attached to d_test-net30 with static ip, d_test-net8 with none ip,\r
  58         none existing danmnet(error)\r
  59         danmnet_pods12: pod attached to d_test-net30 with static ip, d_test-net8 with dynamic ip,\r
  60         d_test-net25 with none ip\r
  61         danmnet_pods13: pod attached to d_test-net8 with static ip, d_test-net24(flannel) with dynamic ip,\r
  62         none existing danmnet(error)\r
  63         danmnet_pods14: pod attached to d_test-net25 with static ip, d_test-net24(flannel) with dynamic ip\r
  64 \r
  65 \r
  66     """\r
  67     steps = ['step1', 'step2', 'step3', 'step5', 'step7', 'step8', 'step9', 'step10', 'step11', 'step12', 'step13',\r
  68              'step14', 'tc_003_clusternetwork_pod_check.Teardown']\r
  69 \r
  70     BuiltIn().run_keyword("tc_003_clusternetwork_pod_check.Setup")\r
  71     common_utils.keyword_runner(steps)\r
  72 \r
  73 \r
  74 def Setup():\r
  75     # execute.execute_unix_command("kubectl create -f /tmp/clusternetwork-test/templates/cnet_attach.yaml")\r
  76     network_attach_test = common_utils.get_helm_chart_content("default/network-attach-test")\r
  77     compare_test_data(network_attach_test, network_attach_properties)\r
  78     replace_ifaces_in_fetched_chart_templates("/tmp/network-attach-test/templates/*")\r
  79     danm_utils.create_resources_from_fetched_chart_templates("/tmp/network-attach-test/templates")\r
  80 \r
  81     install_chart(danmnet_pods1)\r
  82     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods1,\r
  83                                                 expected_result=danmnet_pods1['obj_count'],\r
  84                                                 filter=r'(Running)\s*[0]',\r
  85                                                 timeout=90)\r
  86     install_chart(danmnet_pods2)\r
  87     install_chart(danmnet_pods3)\r
  88     # install_chart(danmnet_pods4)\r
  89     install_chart(danmnet_pods5)\r
  90     install_chart(danmnet_pods6)\r
  91     install_chart(danmnet_pods7)\r
  92     install_chart(danmnet_pods8)\r
  93     install_chart(danmnet_pods9)\r
  94     install_chart(danmnet_pods10)\r
  95 \r
  96 \r
  97 def Teardown():\r
  98     common_utils.helm_delete("danmnet-pods12")\r
  99     common_utils.helm_delete("danmnet-pods14")\r
 100     danm_utils.delete_resources_by_manifest_path("/tmp/network-attach-test/templates/")\r
 101 \r
 102 \r
 103 def step1():\r
 104     # Install danmnet_pods1: all of the pods should be in Running state, check static ips, mac address\r
 105     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods1,\r
 106                                                 expected_result=danmnet_pods1['obj_count'],\r
 107                                                 filter=r'(Running)\s*[0]',\r
 108                                                 timeout=90,\r
 109                                                 delay=10)\r
 110     pod_list = get_pod_list(danmnet_pods1)\r
 111     danmnet_pods1['ip_list'] = get_pod_ips(pod_list)\r
 112     if set(danmnet_pods1['ip_list']) != set(static_ips):\r
 113         raise Exception("Static ip allocation for danmnet-pods1 was not successful, assigned ips!")\r
 114     logger.info("Static ip allocation for danmnet-pods1 was successful")\r
 115     check_mac_address(pod_list, 'cnet_pod1')\r
 116 \r
 117 \r
 118 def step2():\r
 119     # Install danmnet_pods2: ips already used from allocation pool -> 3 pods in containercreating state, check remaining\r
 120     # assigned ips in allocation pool\r
 121     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods2,\r
 122                                                 expected_result=danmnet_pods2['obj_count'],\r
 123                                                 filter=r'(ContainerCreating)\s*[0]',\r
 124                                                 timeout=90)\r
 125     pod_list = get_pod_list(danmnet_pods2)\r
 126     alloc_pool = get_alloc_pool('cnet_pod1', network_attach_properties, 'clusternetwork')\r
 127     danmnet_pods2['ip_list'] = get_pod_ips(pod_list)\r
 128     check_dynamic_ips(alloc_pool, danmnet_pods2['ip_list'])\r
 129 \r
 130 \r
 131 def step3():\r
 132     # Danmnet_pods3 pods are not running because static ips are already allocated\r
 133     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods3,\r
 134                                                 expected_result=danmnet_pods3['obj_count'],\r
 135                                                 filter=r'(ContainerCreating)\s*[0]',\r
 136                                                 timeout=90)\r
 137     # Delete danmnet_pods1, danmnet_pods2\r
 138     common_utils.helm_delete("danmnet-pods2")\r
 139     common_utils.check_kubernetes_object(kube_object=danmnet_pods2,\r
 140                                          tester_function=common_utils.test_kubernetes_object_not_available,\r
 141                                          timeout=90)\r
 142     common_utils.helm_delete("danmnet-pods1")\r
 143 \r
 144 \r
 145 def step5():\r
 146     # Check danmnet_pods1, danmnet_pods2 are purged, ips are reallocated for danmnet_pods3\r
 147 \r
 148     common_utils.check_kubernetes_object(kube_object=danmnet_pods1,\r
 149                                          tester_function=common_utils.test_kubernetes_object_not_available,\r
 150                                          timeout=90)\r
 151     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods3,\r
 152                                                 expected_result=danmnet_pods3['obj_count'],\r
 153                                                 filter=r'(Running)\s*[0]',\r
 154                                                 timeout=60)\r
 155     pod_list = get_pod_list(danmnet_pods3)\r
 156     assigned_ips = get_pod_ips(pod_list, skip_restarts=True)\r
 157     if set(assigned_ips) != set(static_ips):\r
 158         raise Exception("Static ip allocation for danmnet-pods3 was not successful!")\r
 159     logger.info("Static ip allocation for danmnet-pods3 was successful")\r
 160     check_static_routes(pod_list, 'cnet_pod1')\r
 161 \r
 162     actual_pod = list(pod_list)[0]\r
 163     check_connectivity(pod_list, actual_pod, static_ips)\r
 164     actual_pod = list(pod_list)[3]\r
 165     check_connectivity(pod_list, actual_pod, static_ips)\r
 166 \r
 167 \r
 168 def step6():\r
 169     common_utils.check_kubernetes_object(kube_object=danmnet_pods4,\r
 170                                          tester_function=common_utils.test_kubernetes_object_not_available,\r
 171                                          timeout=90)\r
 172     check_danmnet_endpoints(danmnet_pods4, 'test-net2', danmnet_pods4['ip_list'])\r
 173 \r
 174 \r
 175 def step7():\r
 176     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods5,\r
 177                                                 expected_result=danmnet_pods5['obj_count'],\r
 178                                                 filter=r'(ContainerCreating)\s*[0]',\r
 179                                                 timeout=90)\r
 180 \r
 181 \r
 182 def step8():\r
 183     # Dynamic ip allocation fails if no CIDR/allocation pool defined\r
 184     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods6,\r
 185                                                 expected_result=danmnet_pods6['obj_count'],\r
 186                                                 filter=r'(ContainerCreating)\s*[0]',\r
 187                                                 timeout=90)\r
 188 \r
 189 \r
 190 def step9():\r
 191     # multiple interfaces, check flannel and ipvlan ip allocation\r
 192     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods7,\r
 193                                                 expected_result=danmnet_pods7['obj_count'],\r
 194                                                 filter=r'(Running)\s*[0]',\r
 195                                                 timeout=90)\r
 196     pod_list = get_pod_list(danmnet_pods7)\r
 197     assigned_ips = get_pod_ips(pod_list)\r
 198     check_dynamic_ips(flannel_pool, assigned_ips)\r
 199 \r
 200     alloc_pool = get_alloc_pool('cnet_pod3', network_attach_properties, 'clusternetwork')\r
 201     assigned_ips = get_pod_ips(pod_list, if_name='')\r
 202     check_dynamic_ips(alloc_pool, assigned_ips)\r
 203 \r
 204 \r
 205 def step10():\r
 206     # Check service is reachable with flannel\r
 207     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods8,\r
 208                                                 expected_result=danmnet_pods8['obj_count'],\r
 209                                                 filter=r'(Running)\s*[0]',\r
 210                                                 timeout=90)\r
 211     command = "curl danmnet-pods8-1.default.svc.rec.io:4242"\r
 212     res = execute.execute_unix_command_as_root(command)\r
 213     if "OK" not in res:\r
 214         raise Exception("NOK: danmnet-pods8-1 service is not reachable")\r
 215     logger.info("OK: danmnet-pods8-1 service is reachable")\r
 216     pod_list = get_pod_list(danmnet_pods8)\r
 217     assigned_ips = get_pod_ips(pod_list)\r
 218     check_dynamic_ips(flannel_pool, assigned_ips)\r
 219 \r
 220 \r
 221 def step11():\r
 222     # Static ip allocation is ignored with flannel\r
 223     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods9,\r
 224                                                 expected_result=danmnet_pods9['obj_count'],\r
 225                                                 filter=r'(Running)\s*[0]',\r
 226                                                 timeout=90)\r
 227 \r
 228 \r
 229 def step12():\r
 230     # None ip, pod is restarting\r
 231     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods10,\r
 232                                                 expected_result=danmnet_pods10['obj_count'],\r
 233                                                 filter=r'(ContainerCreating)\s*[0]',\r
 234                                                 timeout=90)\r
 235     common_utils.helm_delete("danmnet-pods3")\r
 236     common_utils.helm_delete("danmnet-pods4")\r
 237     common_utils.helm_delete("danmnet-pods5")\r
 238     common_utils.helm_delete("danmnet-pods6")\r
 239     common_utils.helm_delete("danmnet-pods7")\r
 240     common_utils.helm_delete("danmnet-pods8")\r
 241     common_utils.helm_delete("danmnet-pods9")\r
 242     common_utils.helm_delete("danmnet-pods10")\r
 243     common_utils.check_kubernetes_object(kube_object=danmnet_pods_all,\r
 244                                          tester_function=common_utils.test_kubernetes_object_not_available,\r
 245                                          timeout=20)\r
 246     check_dep_count(danmnet_pods1["namespace"], exp_count=0)\r
 247 \r
 248 \r
 249 @pabot_lock("pv_test_ip")\r
 250 @pabot_lock("flannel_ip")\r
 251 @pabot_lock("flannel_ip2")\r
 252 def step13():\r
 253     # danmnet_pods11, danmnet_pods13 has invalid networks attached hance the pod creation will fail,\r
 254     # checking if danmnet endpoints, ips are cleared after several unsuccessful pod creations\r
 255     alloc_before_cnet_pod5 = get_alloc_value('cnet_pod5', network_attach_properties, 'clusternetwork')\r
 256     alloc_before_cnet_pod6 = get_alloc_value('cnet_pod6', network_attach_properties, 'clusternetwork')\r
 257     common_utils.get_helm_chart_content('default/' + danmnet_pods11['obj_name'])\r
 258     execute.execute_unix_command("sed -i 's/{{ .Values.registry_url }}/" + reg + "/g' " + "/tmp/" +\r
 259                                  danmnet_pods11['obj_name'] + "/templates/" + danmnet_pods11['obj_name'] + ".yaml")\r
 260     common_utils.get_helm_chart_content('default/' + danmnet_pods13['obj_name'])\r
 261     execute.execute_unix_command("sed -i 's/{{ .Values.registry_url }}/" + reg + "/g' " + "/tmp/" +\r
 262                                  danmnet_pods13['obj_name'] + "/templates/" + danmnet_pods13['obj_name'] + ".yaml")\r
 263     command = "ls -rt /var/lib/cni/networks/cbr0/ | wc -l"\r
 264     ip_count_before = execute.execute_unix_command_as_root(command)\r
 265     command = "ls -rt /var/lib/cni/networks/cbr0/"\r
 266     cbr0_content1 = execute.execute_unix_command_as_root(command)\r
 267 \r
 268     for _ in range(0, 10):\r
 269         # danmnet_pods11 creation fails\r
 270         command = "kubectl create -f /tmp/" + danmnet_pods11['obj_name'] + "/templates"\r
 271         execute.execute_unix_command_as_root(command)\r
 272 \r
 273         # danmnet_pods13 creation fails\r
 274         command = "kubectl create -f /tmp/" + danmnet_pods13['obj_name'] + "/templates"\r
 275         execute.execute_unix_command_as_root(command)\r
 276         common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods11,\r
 277                                                     expected_result=danmnet_pods11['obj_count'],\r
 278                                                     filter=r'(ContainerCreating)\s*[0]',\r
 279                                                     timeout=40)\r
 280         common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods13,\r
 281                                                     expected_result=danmnet_pods13['obj_count'],\r
 282                                                     filter=r'(ContainerCreating)\s*[0]',\r
 283                                                     timeout=40)\r
 284         command = "kubectl delete -f /tmp/" + danmnet_pods11['obj_name'] + "/templates"\r
 285         execute.execute_unix_command_as_root(command)\r
 286         command = "kubectl delete -f /tmp/" + danmnet_pods13['obj_name'] + "/templates"\r
 287         execute.execute_unix_command_as_root(command)\r
 288         common_utils.check_kubernetes_object(kube_object=danmnet_pods11,\r
 289                                              tester_function=common_utils.test_kubernetes_object_not_available,\r
 290                                              timeout=40)\r
 291         common_utils.check_kubernetes_object(kube_object=danmnet_pods13,\r
 292                                              tester_function=common_utils.test_kubernetes_object_not_available,\r
 293                                              timeout=40)\r
 294     check_danm_count(ip_count_before, cbr0_content1, 0)\r
 295     logger.info("All flannel ips are cleared")\r
 296     alloc_after_cnet_pod5 = get_alloc_value('cnet_pod5', network_attach_properties, 'clusternetwork')\r
 297     alloc_after_cnet_pod6 = get_alloc_value('cnet_pod6', network_attach_properties, 'clusternetwork')\r
 298     if alloc_after_cnet_pod6 != alloc_before_cnet_pod6:\r
 299         raise Exception("allocation value in cnet-pod6 is not as expected")\r
 300     if alloc_after_cnet_pod5 != alloc_before_cnet_pod5:\r
 301         raise Exception("allocation value in cnet-pod5 is not as expected")\r
 302     check_dep_count('default', exp_count=0)\r
 303 \r
 304 \r
 305 def step14():\r
 306     # Static ip, dynamic ip allocation and none ip in the same pod\r
 307     # Check if the same ips can be allocated, which were failing in step 13\r
 308     install_chart(danmnet_pods12)\r
 309     install_chart(danmnet_pods14)\r
 310     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods12,\r
 311                                                 expected_result=danmnet_pods12['obj_count'],\r
 312                                                 filter=r'(Running)\s*[0]',\r
 313                                                 timeout=90)\r
 314     pod_list = get_pod_list(danmnet_pods12)\r
 315     alloc_pool = get_alloc_pool('cnet_pod6', network_attach_properties, 'clusternetwork')\r
 316     danmnet_pods12['ip_list'] = get_pod_ips(pod_list, if_name='eth1')\r
 317     check_dynamic_ips(alloc_pool, danmnet_pods12['ip_list'])\r
 318     danmnet_pods12['ip_list'] = get_pod_ips(pod_list, if_name='eth0')\r
 319     if IPAddress(danmnet_pods12['ip_list'][0]) != IPAddress('10.10.0.250'):\r
 320         raise Exception("static ip in pod danmnet-pods12 is not as expected")\r
 321 \r
 322     common_utils.test_kubernetes_object_quality(kube_object=danmnet_pods14,\r
 323                                                 expected_result=danmnet_pods14['obj_count'],\r
 324                                                 filter=r'(Running)\s*[0]',\r
 325                                                 timeout=90)\r
 326     pod_list = get_pod_list(danmnet_pods14)\r
 327     danmnet_pods14['ip_list'] = get_pod_ips(pod_list, if_name='eth2')\r
 328     if IPAddress(danmnet_pods14['ip_list'][0]) != IPAddress('10.10.0.254'):\r
 329         raise Exception("static ip in pod danmnet-pods14 is not as expected")\r
 330     common_utils.helm_delete("danmnet-pods12")\r
 331     common_utils.helm_delete("danmnet-pods14")\r
 332     common_utils.check_kubernetes_object(kube_object=danmnet_pods14,\r
 333                                          tester_function=common_utils.test_kubernetes_object_not_available,\r
 334                                          timeout=20)\r
 335     check_dep_count(danmnet_pods12["namespace"], exp_count=0)\r
 336 \r
 337 \r
 338 @robot_log\r
 339 def check_danm_count(ip_count_before_parameter, cbr0_content1_parameter, tries):\r
 340     command = "ls -rt /var/lib/cni/networks/cbr0/"\r
 341     cbr0_content2 = execute.execute_unix_command_as_root(command)\r
 342     if tries == 3:\r
 343         diff = list(set(cbr0_content1_parameter) - set(cbr0_content2))\r
 344         logger.info("Additional IPs after step: " + diff)\r
 345         for ip in diff:\r
 346             command = "cat /var/lib/cni/networks/cbr0/" + ip + " | grep -v eth"\r
 347             cid = execute.execute_unix_command(command)\r
 348             command = "docker ps -a --no-trunc | grep " + cid\r
 349             docker_ps = execute.execute_unix_command(command)\r
 350             logger.info("Additional ip belongs to the following container: " + docker_ps)\r
 351         raise Exception("Flannel ips are not cleared after pod deletion")\r
 352     else:\r
 353         tries = tries + 1\r
 354     command = "ls -rt /var/lib/cni/networks/cbr0/ | wc -l"\r
 355     ip_count_after = execute.execute_unix_command_as_root(command)\r
 356     ip_count_before = ip_count_before_parameter\r
 357     cbr0_content1 = cbr0_content1_parameter\r
 358     if ip_count_before != ip_count_after:\r
 359         logger.info(cbr0_content1)\r
 360         logger.info(cbr0_content2)\r
 361         time.sleep(30)\r
 362         check_danm_count(ip_count_before, cbr0_content1, tries)\r
 363 \r
 364 \r
 365 def install_chart(kube_object):\r
 366     common_utils.helm_install(chart_name="default/" + kube_object['obj_name'], release_name=kube_object['obj_name'])\r
 367 \r
 368 \r
 369 @robot_log\r
 370 def get_pod_ips(pod_list, skip_restarts=False, if_name='eth0'):\r
 371     assigned_ips = []\r
 372     for key in pod_list:\r
 373         if (pod_list[key]['status'] == 'Running') and ((pod_list[key]['restarts'] == '0') or skip_restarts):\r
 374             logger.info(pod_list[key]['namespace'])\r
 375             if if_name != '':\r
 376                 command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + " ip a | grep " + if_name + \\r
 377                           " | grep inet | awk '{print $2}' | awk -F \"/\" '{print $1}' "\r
 378             else:\r
 379                 command = "kubectl exec " + key + " -n " + pod_list[key]['namespace'] + \\r
 380                           "  -- ip -o a | grep -vE '(: lo|: eth0)' | grep inet | awk '{print $4}' | " \\r
 381                           "awk -F \"/\" '{print $1}'"\r
 382             assigned_ips.append(execute.execute_unix_command_as_root(command))\r
 383     return assigned_ips\r
 384 \r
 385 \r
 386 @robot_log\r
 387 def get_alloc_pool(network, dictionary, resource_type):\r
 388     alloc_pool = {}\r
 389     command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -o yaml " + \\r
 390               " | grep allocation_pool -A 2 | grep start | awk {'print$2'}"\r
 391     alloc_pool['start'] = execute.execute_unix_command_as_root(command)\r
 392     command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -o yaml " + \\r
 393               " | grep allocation_pool -A 2 | grep end | awk {'print$2'}"\r
 394     alloc_pool['end'] = execute.execute_unix_command_as_root(command)\r
 395     return alloc_pool\r
 396 \r
 397 \r
 398 @robot_log\r
 399 def get_pod_list(kube_object):\r
 400     pod_list = {}\r
 401     command = "kubectl get pod --all-namespaces | grep -w " + kube_object[\r
 402         'obj_name'] + " | awk '{print $1 \" \" $2 \" \" $4 \" \" $5}'"\r
 403     for line in execute.execute_unix_command_as_root(command).split('\r\n'):\r
 404         pod_list[line.split(' ')[1]] = {'namespace': line.split(' ')[0], 'status': line.split(' ')[2],\r
 405                                         'restarts': line.split(' ')[3]}\r
 406     return pod_list\r
 407 \r
 408 \r
 409 @robot_log\r
 410 def check_dynamic_ips(alloc_pool, assigned_ips):\r
 411     for ip in assigned_ips:\r
 412         if (IPAddress(alloc_pool['start']) > IPAddress(ip)) or (IPAddress(ip) > IPAddress(alloc_pool['end'])):\r
 413             raise Exception("Dynamic ip is not in allocation pool")\r
 414     logger.info("All dynamic ips are from the allocation pool.")\r
 415     if len(list(set(assigned_ips))) != len(assigned_ips):\r
 416         raise Exception("duplicated IPs assigned")\r
 417     logger.info("All allocated IPs are unique")\r
 418 \r
 419 \r
 420 @robot_log\r
 421 def check_static_routes(pod_list, danmnet):\r
 422     for pod in pod_list:\r
 423         if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'):\r
 424             command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " route | grep " + \\r
 425                       network_attach_properties[danmnet]['routes'].split('/')[0] + " | grep " + \\r
 426                       network_attach_properties[danmnet]['routes'].split(' ')[1] + " | wc -l"\r
 427             res = execute.execute_unix_command_as_root(command)\r
 428             if res != '1':\r
 429                 raise Exception("static route in pod " + pod + " does not match with route defined in " + danmnet)\r
 430             logger.info("Static route in pod " + pod + " is as it should be.")\r
 431 \r
 432 \r
 433 @robot_log\r
 434 def check_mac_address(pod_list, network):\r
 435     command = "ip a | grep -wA 1 " + network_attach_properties[network]['host_if'] + " | grep ether | awk '{print $2}'"\r
 436     host_mac = execute.execute_unix_command_as_root(command)\r
 437     for pod in pod_list:\r
 438         if (pod_list[pod]['status'] == 'Running') and (pod_list[pod]['restarts'] == '0'):\r
 439             command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " ip a | grep -A 1 eth0 | " \\r
 440                                                                                     "grep link | awk '{print $2}'"\r
 441             pod_mac = execute.execute_unix_command_as_root(command)\r
 442             if host_mac != pod_mac:\r
 443                 raise Exception("Wrong Mac address in pod " + pod)\r
 444             logger.info("Correct mac address in pod " + pod)\r
 445 \r
 446 \r
 447 @robot_log\r
 448 def check_danmnet_endpoints(kube_object, network, assigned_ips):\r
 449     for ip in assigned_ips:\r
 450         command = "kubectl get danmep -n " + kube_object['namespace'] + " -o yaml | grep -B 10 " + \\r
 451                   network_attach_properties[network]['name'] + " | grep " + ip + " | wc -l"\r
 452         res = execute.execute_unix_command_as_root(command)\r
 453         if res != '0':\r
 454             raise Exception("Endpoint with ip " + ip + " still exists.")\r
 455     logger.info("The necessary endpoints are cleared")\r
 456 \r
 457 \r
 458 @robot_log\r
 459 def check_connectivity(pod_list, pod, ip_list):\r
 460     for ip in ip_list:\r
 461         command = "kubectl exec " + pod + " -n " + pod_list[pod]['namespace'] + " -- sh -c \"ping -c 1 " + ip + "\""\r
 462         stdout = execute.execute_unix_command_as_root(command)\r
 463         if '0% packet loss' not in stdout:\r
 464             raise Exception("pod " + pod + " cannot reach ip " + ip)\r
 465         logger.info("pod " + pod + " can reach ip " + ip)\r
 466 \r
 467 \r
 468 @robot_log\r
 469 def check_dep_count(namespace, exp_count):\r
 470     tries = 0\r
 471     danm_eps = get_deps(namespace)\r
 472     test_pod_name_pattern = r'^danmnet-pods'\r
 473     danmnet_test_deps = [dep for dep in danm_eps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)]\r
 474     while (tries < 5) and (len(danmnet_test_deps) != exp_count):\r
 475         time.sleep(20)\r
 476         tries += 1\r
 477         danm_eps = get_deps(namespace)\r
 478         danmnet_test_deps = [dep for dep in danm_eps if is_dep_belongs_to_pod(dep, test_pod_name_pattern)]\r
 479 \r
 480     if len(danmnet_test_deps) != exp_count:\r
 481         raise Exception("Danm endpoint count is not as expected! Got: " + str(len(danmnet_test_deps)) + ", expected: " +\r
 482                         str(exp_count))\r
 483     logger.info("Danm endpoint count is as expected.")\r
 484 \r
 485 \r
 486 @robot_log\r
 487 def get_deps(namespace):\r
 488     command = "kubectl get dep -n {} -o json".format(namespace)\r
 489     deps_text = execute.execute_unix_command_as_root(command)\r
 490     return json.loads(deps_text).get("items")\r
 491 \r
 492 \r
 493 @robot_log\r
 494 def is_dep_belongs_to_pod(dep, pod_pattern):\r
 495     pod_name = dep["spec"]["Pod"]\r
 496     return bool(re.search(pod_pattern, pod_name))\r
 497 \r
 498 \r
 499 @robot_log\r
 500 def get_alloc_value(network, dictionary, resource_type):\r
 501     command = "kubectl get " + resource_type + " " + dictionary[network]['name'] + " -o yaml | grep -w alloc | " \\r
 502                                                                                    "awk '{print $2}'"\r
 503     alloc = execute.execute_unix_command_as_root(command)\r
 504     return alloc\r
 505 \r
 506 \r
 507 @robot_log\r
 508 def replace_ifaces_in_fetched_chart_templates(path):\r
 509     execute.execute_unix_command("sed -i 's/{{ .Values.infra_int_if }}/" + infra_int_if + "/g' " + path)\r
 510     execute.execute_unix_command("sed -i 's/{{ .Values.infra_ext_if }}/" + infra_ext_if + "/g' " + path)\r
 511     execute.execute_unix_command("sed -i 's/{{ .Values.infra_storage_if }}/" + infra_storage_if + "/g' " + path)\r
 512 \r
 513 \r
 514 @robot_log\r
 515 def compare_test_data(list_to_compare, dict_to_compare):\r
 516     for danmnet in list_to_compare:\r
 517         if danmnet not in dict_to_compare:\r
 518             logger.warn(danmnet + " is not present in test constants: {}".format(dict_to_compare))\r
 519     for key in dict_to_compare:\r
 520         if key not in list_to_compare:\r
 521             logger.warn(key + " is not present in {} chart".format(list_to_compare))\r
 522 \r
 523 \r
 524 @robot_log\r
 525 def delete_all_resources(resource_type):\r
 526     execute.execute_unix_command("kubectl delete " + resource_type + " --all")\r