testcases/danm_network_check/tc_002_tenantnetwork_pod_check.py

   1 import sys
   2 import os
   3 import common_utils
   4 import danm_utils
   5
   6 from execute_command import execute_command
   7 from decorators_for_robot_functionalities import *
   8 from test_constants import *
   9
  10 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '../libraries/common'))
  11
  12 execute = BuiltIn().get_library_instance('execute_command')
  13 stack_infos = BuiltIn().get_library_instance('stack_infos')
  14 infra_int_if = stack_infos.get_infra_int_if()
  15 infra_ext_if = stack_infos.get_infra_ext_if()
  16 infra_storage_if = stack_infos.get_infra_storage_if()
  17
  18
  19 def tc_002_tenantnetwork_pod_check():
  20     steps = ['step1_check_static_ip_allocations',
  21              'step2_check_dynamic_ip_shortage',
  22              'step3_check_static_ip_shortage',
  23              'step4_check_attach_in_kubesystem_namespace',
  24              'step5_check_static_ip_alloc_static_routes_success_after_purge',
  25              'step6_check_step4_deletion_success',
  26              'step7_check_static_ip_alloc_outside_cidr',
  27              'step8_check_ip_alloc_with_cidrless_allocpoolless_tenantnet',
  28              'step9_check_connection_to_flannel_and_ipvlan_tenantnetworks',
  29              'step10_check_service_reachability_with_flannel',
  30              'step11_check_flannel_static_ip_alloc_not_in_flannel_cidr_ignored',
  31              'step12_none_ip_pod_restart_loop',
  32              'step13_check_invalid_net_attach_and_successful_damnep_ip_release_after_retries',
  33              'step14_check_realloc_ips_of_prev_step_with_dynamic_and_none_ip_alloc',
  34              'tc_002_tenantnetwork_pod_check.Teardown']
  35
  36     BuiltIn().run_keyword("tc_002_tenantnetwork_pod_check.Setup")
  37     common_utils.keyword_runner(steps)
  38
  39
  40 @pabot_lock("health_check_1")
  41 def Setup():
  42     tennet_attach_test = common_utils.get_helm_chart_content("default/tenantnetwork-attach-test")
  43     danm_utils.compare_test_data(tennet_attach_test, tenantnetwork_attach_properties)
  44
  45     replace_ifaces_in_fetched_chart_templates("/tmp/tenantnetwork-attach-test/templates/*")
  46     replace_ifaces_in_fetched_chart_templates("/tmp/tenantconfig-test/templates/*")
  47     # deploy a valid TenantConfig
  48     danm_utils.create_resources_from_fetched_chart_templates("/tmp/tenantconfig-test/templates/tconf_05.yaml")
  49     # remove the default TenantConfig
  50     execute.execute_unix_command("kubectl delete tenantconfig danm-tenant-config")
  51     # deploy all TenantNetwork-s
  52     danm_utils.create_resources_from_fetched_chart_templates("/tmp/tenantnetwork-attach-test/templates")
  53     set_expected_host_if_in_constants(tenantnetwork_attach_properties)
  54
  55
  56 @pabot_lock("health_check_1")
  57 def Teardown():
  58     execute.execute_unix_command("kubectl create -f /var/lib/caas/manifests/danm-tenant-config.yaml")
  59     execute.execute_unix_command("kubectl delete tenantconfig tconf-05")
  60     execute.execute_unix_command("kubectl delete -f /tmp/tenantnetwork-attach-test/templates/")
  61
  62
  63 def step1_check_static_ip_allocations():
  64     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod1", release_name="tenantnetwork-attach-pod1")
  65     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod1,
  66                                                 expected_result=tennet_pod1['obj_count'],
  67                                                 filter=r'(Running)\s*[0]',
  68                                                 timeout=60,
  69                                                 delay=10)
  70     pod_list = danm_utils.get_pod_list(tennet_pod1)
  71     if set(tennet_pod1['ip_list']) != set(danm_utils.get_pod_ips(pod_list)):
  72         raise Exception("Static ip allocation for tenantnetwork-attach-pod1 was unsuccessful!")
  73     logger.info("Static ips allocated successfully!")
  74     danm_utils.check_mac_address(pod_list, 'tennet_attach_01', tenantnetwork_attach_properties)
  75
  76
  77 def step2_check_dynamic_ip_shortage():
  78     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod2", release_name="tenantnetwork-attach-pod2")
  79     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod2,
  80                                                 expected_result=tennet_pod2['obj_count'],
  81                                                 filter=r'(ContainerCreating)\s*[0]',
  82                                                 timeout=60)
  83     alloc_pool = danm_utils.get_alloc_pool('tennet_attach_01', tenantnetwork_attach_properties, 'tenantnetwork')
  84     danm_utils.check_dynamic_ips(alloc_pool, tennet_pod2['ip_list'])
  85
  86
  87 def step3_check_static_ip_shortage():
  88     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod3", release_name="tenantnetwork-attach-pod3")
  89     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod3,
  90                                                 expected_result=tennet_pod3['obj_count'],
  91                                                 filter=r'(ContainerCreating)\s*[0]',
  92                                                 timeout=30)
  93     common_utils.helm_delete("tenantnetwork-attach-pod2")
  94     common_utils.check_kubernetes_object(kube_object=tennet_pod2,
  95                                          tester_function=common_utils.test_kubernetes_object_not_available,
  96                                          timeout=60)
  97     common_utils.helm_delete("tenantnetwork-attach-pod1")
  98
  99
 100 def step4_check_attach_in_kubesystem_namespace():
 101     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod4", release_name="tenantnetwork-attach-pod4")
 102     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod4,
 103                                                 expected_result=tennet_pod4['obj_count'],
 104                                                 filter=r'(Running)\s*[0]',
 105                                                 timeout=60,
 106                                                 delay=10)
 107     alloc_pool = danm_utils.get_alloc_pool("tennet_attach_02", tenantnetwork_attach_properties, 'tenantnetwork')
 108     danm_utils.check_dynamic_ips(alloc_pool, tennet_pod4['ip_list'])
 109     common_utils.helm_delete(release_name="tenantnetwork-attach-pod4")
 110
 111
 112 def step5_check_static_ip_alloc_static_routes_success_after_purge():
 113     common_utils.check_kubernetes_object(kube_object=tennet_pod1,
 114                                          tester_function=common_utils.test_kubernetes_object_not_available,
 115                                          timeout=60)
 116     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod3,
 117                                                 expected_result=tennet_pod3['obj_count'],
 118                                                 filter=r'(Running)\s*[0]',
 119                                                 timeout=60)
 120     pod_list = danm_utils.get_pod_list(tennet_pod3)
 121     if set(tennet_pod3['ip_list']) != set(danm_utils.get_pod_ips(pod_list)):
 122         raise Exception("Static ip allocation for tenantnetwork-attach-pod3 was unsuccessful!")
 123     logger.info("Static ips allocated successfully!")
 124
 125     danm_utils.check_static_routes(pod_list, 'tennet_attach_01', tenantnetwork_attach_properties)
 126
 127     danm_utils.check_connectivity(pod_list, list(pod_list)[0], tennet_pod3['ip_list'])
 128     danm_utils.check_connectivity(pod_list, list(pod_list)[3], tennet_pod3['ip_list'])
 129
 130
 131 @robot_log
 132 def step6_check_step4_deletion_success():
 133     common_utils.check_kubernetes_object(kube_object=tennet_pod4,
 134                                          tester_function=common_utils.test_kubernetes_object_not_available,
 135                                          timeout=60)
 136     danm_utils.check_danmnet_endpoints_deleted(tennet_pod4, 'tennet_attach_02', tenantnetwork_attach_properties,
 137                                                tennet_pod4['ip_list'])
 138
 139
 140 @robot_log
 141 def step7_check_static_ip_alloc_outside_cidr():
 142     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod5", release_name="tenantnetwork-attach-pod5")
 143     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod5,
 144                                                 expected_result=tennet_pod5['obj_count'],
 145                                                 filter=r'(ContainerCreating)\s*[0]',
 146                                                 timeout=90)
 147
 148
 149 @robot_log
 150 def step8_check_ip_alloc_with_cidrless_allocpoolless_tenantnet():
 151     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod6", release_name="tenantnetwork-attach-pod6")
 152     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod6,
 153                                                 expected_result=tennet_pod6['obj_count'],
 154                                                 filter=r'(ContainerCreating)\s*[0]',
 155                                                 timeout=90)
 156
 157
 158 @robot_log
 159 def step9_check_connection_to_flannel_and_ipvlan_tenantnetworks():
 160     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod7", release_name="tenantnetwork-attach-pod7")
 161     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod7,
 162                                                 expected_result=tennet_pod7['obj_count'],
 163                                                 filter=r'(Running)\s*[0]',
 164                                                 timeout=90)
 165
 166     pod_list = danm_utils.get_pod_list(tennet_pod7)
 167     danm_utils.check_dynamic_ips(tenantnetwork_attach_properties['tennet_attach_04']['flannel_pool'],
 168                                  danm_utils.get_pod_ips(pod_list))
 169
 170     alloc_pool = danm_utils.get_alloc_pool('tennet_attach_03', tenantnetwork_attach_properties, 'tenantnetwork')
 171     danm_utils.check_dynamic_ips(alloc_pool, danm_utils.get_pod_ips(pod_list, if_name=''))
 172
 173
 174 @robot_log
 175 def step10_check_service_reachability_with_flannel():
 176     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod8", release_name="tenantnetwork-attach-pod8")
 177     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod8,
 178                                                 expected_result=tennet_pod8['obj_count'],
 179                                                 filter=r'(Running)\s*[0]',
 180                                                 timeout=90)
 181     command = "curl tennet-pod-08.default.svc.rec.io:4242"
 182     res = execute.execute_unix_command_as_root(command)
 183     if "OK" not in res:
 184         raise Exception("NOK: tennet-pod-08 service is not reachable")
 185     logger.info("OK: tennet-pod-08 service is reachable")
 186     pod_list = danm_utils.get_pod_list(tennet_pod8)
 187     assigned_ips = danm_utils.get_pod_ips(pod_list)
 188     danm_utils.check_dynamic_ips(tenantnetwork_attach_properties['tennet_attach_04']['flannel_pool'], assigned_ips)
 189
 190
 191 def step11_check_flannel_static_ip_alloc_not_in_flannel_cidr_ignored():
 192     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod9", release_name="tenantnetwork-attach-pod9")
 193     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod9,
 194                                                 expected_result=tennet_pod9['obj_count'],
 195                                                 filter=r'(Running)\s*[0]',
 196                                                 timeout=90)
 197
 198
 199 def step12_none_ip_pod_restart_loop():
 200     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod10",
 201                               release_name="tenantnetwork-attach-pod10")
 202     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod10,
 203                                                 expected_result=tennet_pod10['obj_count'],
 204                                                 filter=r'(ContainerCreating)\s*[0]',
 205                                                 timeout=90)
 206     common_utils.helm_delete("tenantnetwork-attach-pod3")
 207     common_utils.helm_delete("tenantnetwork-attach-pod5")
 208     common_utils.helm_delete("tenantnetwork-attach-pod6")
 209     common_utils.helm_delete("tenantnetwork-attach-pod7")
 210     common_utils.helm_delete("tenantnetwork-attach-pod8")
 211     common_utils.helm_delete("tenantnetwork-attach-pod9")
 212     common_utils.helm_delete("tenantnetwork-attach-pod10")
 213
 214
 215 def step13_check_invalid_net_attach_and_successful_damnep_ip_release_after_retries():
 216     tnet1_alloc_before = danm_utils.get_alloc_value('tennet_attach_01', tenantnetwork_attach_properties,
 217                                                     "tenantnetwork")
 218     tnet5_alloc_before = danm_utils.get_alloc_value('tennet_attach_05', tenantnetwork_attach_properties,
 219                                                     "tenantnetwork")
 220     tnet6_alloc_before = danm_utils.get_alloc_value('tennet_attach_06', tenantnetwork_attach_properties,
 221                                                     "tenantnetwork")
 222     common_utils.get_helm_chart_content("default/tenantnetwork-attach-pod11")
 223     common_utils.get_helm_chart_content("default/tenantnetwork-attach-pod13")
 224
 225     execute.execute_unix_command("sed -i 's/{{ .Values.registry_url }}/" + reg + "/g' " +
 226                                  "/tmp/tenantnetwork-attach-pod11/templates/tennet_pod_11.yaml")
 227     execute.execute_unix_command("sed -i 's/{{ .Values.registry_url }}/" + reg + "/g' " +
 228                                  "/tmp/tenantnetwork-attach-pod13/templates/tennet_pod_13.yaml")
 229
 230     for _ in range(0, 10):
 231         danm_utils.create_resources_from_fetched_chart_templates("/tmp/tenantnetwork-attach-pod11/templates")
 232         danm_utils.create_resources_from_fetched_chart_templates("/tmp/tenantnetwork-attach-pod13/templates")
 233         common_utils.test_kubernetes_object_quality(kube_object=tennet_pod11,
 234                                                     expected_result=tennet_pod11['obj_count'],
 235                                                     filter=r'(ContainerCreating)\s*[0]',
 236                                                     timeout=40)
 237         common_utils.test_kubernetes_object_quality(kube_object=tennet_pod13,
 238                                                     expected_result=tennet_pod13['obj_count'],
 239                                                     filter=r'(ContainerCreating)\s*[0]',
 240                                                     timeout=40)
 241         danm_utils.delete_resources_by_manifest_path("/tmp/tenantnetwork-attach-pod11/templates")
 242         danm_utils.delete_resources_by_manifest_path("/tmp/tenantnetwork-attach-pod13/templates")
 243         common_utils.check_kubernetes_object(kube_object=tennet_pod11,
 244                                              tester_function=common_utils.test_kubernetes_object_not_available,
 245                                              timeout=40)
 246         common_utils.check_kubernetes_object(kube_object=tennet_pod13,
 247                                              tester_function=common_utils.test_kubernetes_object_not_available,
 248                                              timeout=40)
 249
 250     tnet1_alloc_after = danm_utils.get_alloc_value('tennet_attach_01', tenantnetwork_attach_properties, 'tenantnetwork')
 251     tnet5_alloc_after = danm_utils.get_alloc_value('tennet_attach_05', tenantnetwork_attach_properties, 'tenantnetwork')
 252     tnet6_alloc_after = danm_utils.get_alloc_value('tennet_attach_06', tenantnetwork_attach_properties, 'tenantnetwork')
 253     if tnet1_alloc_before != tnet1_alloc_after:
 254         raise Exception("allocation value in tennet_attach_01 is not as expected")
 255     if tnet5_alloc_before != tnet5_alloc_after:
 256         raise Exception("allocation value in tennet_attach_05 is not as expected")
 257     if tnet6_alloc_before != tnet6_alloc_after:
 258         raise Exception("allocation value in tennet_attach_06 is not as expected")
 259     danm_utils.check_dep_count('default', exp_count=0)
 260
 261
 262 def step14_check_realloc_ips_of_prev_step_with_dynamic_and_none_ip_alloc():
 263     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod12",
 264                               release_name="tenantnetwork-attach-pod12")
 265     common_utils.helm_install(chart_name="default/tenantnetwork-attach-pod14",
 266                               release_name="tenantnetwork-attach-pod14")
 267     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod12,
 268                                                 expected_result=tennet_pod12['obj_count'],
 269                                                 filter=r'(Running)\s*[0]',
 270                                                 timeout=90)
 271     pod_list = danm_utils.get_pod_list(tennet_pod12)
 272     alloc_pool = danm_utils.get_alloc_pool("tennet_attach_01", tenantnetwork_attach_properties, "tenantnetwork")
 273     danm_utils.check_dynamic_ips(alloc_pool, danm_utils.get_pod_ips(pod_list, if_name='tnet_1'))
 274     alloc_pool = danm_utils.get_alloc_pool("tennet_attach_05", tenantnetwork_attach_properties, "tenantnetwork")
 275     danm_utils.check_dynamic_ips(alloc_pool, danm_utils.get_pod_ips(pod_list, if_name='eth0'))
 276
 277     common_utils.test_kubernetes_object_quality(kube_object=tennet_pod14,
 278                                                 expected_result=tennet_pod14['obj_count'],
 279                                                 filter=r'(Running)\s*[0]',
 280                                                 timeout=90)
 281     pod_list = danm_utils.get_pod_list(tennet_pod14)
 282     # danm_utils.check_dynamic_ips(alloc_pool, [tennet_pod14['ip_list'][2]])
 283     danm_utils.check_dynamic_ips(alloc_pool, danm_utils.get_pod_ips(pod_list, if_name='tnet5'))
 284     alloc_pool = danm_utils.get_alloc_pool("tennet_attach_06", tenantnetwork_attach_properties, "tenantnetwork")
 285     danm_utils.check_dynamic_ips(alloc_pool, danm_utils.get_pod_ips(pod_list, if_name='eth0'))
 286     alloc_pool = danm_utils.get_alloc_pool("tennet_attach_01", tenantnetwork_attach_properties, "tenantnetwork")
 287     danm_utils.check_dynamic_ips(alloc_pool, danm_utils.get_pod_ips(pod_list, if_name='tnet_2'))
 288     common_utils.helm_delete("tenantnetwork-attach-pod12")
 289     common_utils.helm_delete("tenantnetwork-attach-pod14")
 290     common_utils.check_kubernetes_object(kube_object=tennet_pod12,
 291                                          tester_function=common_utils.test_kubernetes_object_not_available,
 292                                          timeout=20)
 293     common_utils.check_kubernetes_object(kube_object=tennet_pod14,
 294                                          tester_function=common_utils.test_kubernetes_object_not_available,
 295                                          timeout=20)
 296     danm_utils.check_dep_count(tennet_pod12["namespace"], exp_count=0)
 297
 298
 299 @robot_log
 300 def replace_ifaces_in_fetched_chart_templates(path):
 301     execute.execute_unix_command("sed -i 's/{{ .Values.infra_int_if }}/" + infra_int_if + "/g' " + path)
 302     execute.execute_unix_command("sed -i 's/{{ .Values.infra_ext_if }}/" + infra_ext_if + "/g' " + path)
 303     execute.execute_unix_command("sed -i 's/{{ .Values.infra_storage_if }}/" + infra_storage_if + "/g' " + path)
 304
 305
 306 # TODO: figure out sg for host_if verif, make all v(x)lan and fill expected res in prop_dict
 307 @robot_log
 308 def set_expected_host_if_in_constants(properties_dict):
 309     for elem in properties_dict:
 310         properties_dict[elem]['host_if'] = infra_int_if