testcases/cpu_pooling/tc_002_exclusive_pool_tests.py

   1 import sys
   2 import os
   3 from robot.libraries.BuiltIn import BuiltIn
   4 from robot.libraries.String import String
   5 from robot.api import logger
   6 from decorators_for_robot_functionalities import *
   7 from time import sleep
   8 from test_constants import *
   9
  10 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '../libraries/common'))
  11 import common_utils  # noqa
  12
  13
  14 ex = BuiltIn().get_library_instance('execute_command')
  15 cpupools = {}
  16
  17
  18 def tc_002_exclusive_pool_tests():
  19     steps = [
  20         'step1_no_annotation',
  21         'step2_with_annotation',
  22         'step3_more_replicas_than_cpus',
  23         'step4_request_more_than_cpus',
  24         'step5_less_cpu_annotation_than_request',
  25         'step6_more_cpu_annotation_than_request',
  26         'step_7_allocate_all_exclusive_and_new_one_start_running_after_needed_resource_is_freed_up'
  27     ]
  28
  29     BuiltIn().run_keyword("tc_002_exclusive_pool_tests.Setup")
  30     common_utils.keyword_runner(steps)
  31
  32
  33 def Setup():
  34     global cpupools
  35     global nodename
  36     nodename = common_utils.decide_nodename()
  37     cpupools = common_utils.get_cpupools()
  38     logger.info("CPU pools: " + str(cpupools))
  39     logger.info("Default nodename to deploy: " + nodename)
  40
  41
  42 @pabot_lock("flannel_ip")
  43 def step1_no_annotation():
  44     try:
  45         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive1", release_name="cpu-pooling",
  46                                   values="registry_url={reg_url},nodename={node_name}".format(reg_url=reg,
  47                                                                                               node_name=nodename))
  48         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod1,
  49                                                     expected_result="1",
  50                                                     filter=r'(Running)\s*[0]',
  51                                                     timeout=90)
  52
  53         allowed_cpu_for_pod = common_utils.get_cpu_allowed_list_from_pod(cpu_pooling_pod1['obj_name'])
  54         exclusive_cpus = cpupools[nodename]['exclusive_caas']
  55         if not common_utils.allowed_cpus_is_in_cpu_pool(allowed_cpu_for_pod, exclusive_cpus):
  56             raise Exception('{pod} not allocate CPU from exclusive pool!'.format(pod=cpu_pooling_pod1['obj_name']))
  57     except Exception as e:
  58         raise e
  59     finally:
  60         common_utils.helm_delete("cpu-pooling")
  61         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod1,
  62                                              tester_function=common_utils.test_kubernetes_object_not_available,
  63                                              timeout=90)
  64
  65
  66 @pabot_lock("flannel_ip")
  67 def step2_with_annotation():
  68     try:
  69         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive2", release_name="cpu-pooling",
  70                                   values="registry_url={reg_url},nodename={node_name}".format(reg_url=reg,
  71                                                                                               node_name=nodename))
  72         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod2,
  73                                                     expected_result="1",
  74                                                     filter=r'(Running)\s*[0]',
  75                                                     timeout=90)
  76
  77         allowed_cpu_for_pod = common_utils.get_cpu_allowed_list_from_pod(cpu_pooling_pod2['obj_name'])
  78         exclusive_cpus = cpupools[nodename]['exclusive_caas']
  79         if not common_utils.allowed_cpus_is_in_cpu_pool(allowed_cpu_for_pod, exclusive_cpus):
  80             raise Exception('{pod} not allocate CPU from exclusive pool!'.format(pod=cpu_pooling_pod2['obj_name']))
  81     finally:
  82         common_utils.helm_delete("cpu-pooling")
  83         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
  84                                              tester_function=common_utils.test_kubernetes_object_not_available,
  85                                              timeout=90)
  86
  87
  88 @pabot_lock("flannel_ip")
  89 def step3_more_replicas_than_cpus():
  90     num_of_replicas = len(cpupools[nodename]['exclusive_caas'])
  91     try:
  92         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive2", release_name="cpu-pooling",
  93                                   values="registry_url={reg_url},nodename={node_name},replicas={cpus}"
  94                                   .format(reg_url=reg, cpus=num_of_replicas+1, node_name=nodename))
  95         cpu_pooling_pod2['obj_count'] = str(num_of_replicas)
  96         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod2,
  97                                                     expected_result="1",
  98                                                     filter=r'(Pending)\s*[0]',
  99                                                     timeout=90,
 100                                                     delay=3)
 101         result = ex.execute_unix_command('kubectl describe pod {podname}'.format(podname=cpu_pooling_pod2['obj_name']))
 102         error = 'Insufficient nokia.k8s.io/exclusive_caas'
 103
 104         if error not in result:
 105             raise Exception('Replicaset description does not contain expected error! -' + result)
 106         else:
 107             logger.info(error)
 108     finally:
 109         cpu_pooling_pod2['obj_count'] = "1"
 110
 111         common_utils.helm_delete("cpu-pooling")
 112         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 113                                              tester_function=common_utils.test_kubernetes_object_not_available,
 114                                              timeout=90)
 115
 116
 117 @pabot_lock("flannel_ip")
 118 def step4_request_more_than_cpus():
 119     max_exclusive_pool_size = len(cpupools[nodename]['exclusive_caas'])
 120     try:
 121         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive2", release_name="cpu-pooling",
 122                                   values="registry_url={reg_url},nodename={node_name},proc_req={cpus},pool_req={cpus}"
 123                                   .format(reg_url=reg, cpus=max_exclusive_pool_size+1, node_name=nodename))
 124         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod2,
 125                                                     expected_result="1",
 126                                                     filter=r'(Pending)\s*[0]',
 127                                                     timeout=90,
 128                                                     delay=3)
 129         result = ex.execute_unix_command('kubectl describe pod {podname}'.format(podname=cpu_pooling_pod2['obj_name']))
 130         error = 'Insufficient nokia.k8s.io/exclusive_caas'
 131
 132         if error not in result:
 133             raise Exception('Replicaset description does not contain expected error! -' + result)
 134         else:
 135             logger.info(error)
 136     finally:
 137         common_utils.helm_delete("cpu-pooling")
 138         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 139                                              tester_function=common_utils.test_kubernetes_object_not_available,
 140                                              timeout=90)
 141
 142
 143 @pabot_lock("flannel_ip")
 144 def step5_less_cpu_annotation_than_request():
 145     annotation_cpu = 1
 146     request_cpu = 2
 147     cpu_pooling_pod2['obj_type'] = 'replicaset'
 148     try:
 149         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive2", release_name="cpu-pooling",
 150                                   values="registry_url={url},nodename={node_name},proc_req={proc},pool_req={req}"
 151                                   .format(url=reg, proc=annotation_cpu, req=request_cpu, node_name=nodename))
 152         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 153                                              tester_function=common_utils.test_kubernetes_object_available,
 154                                              timeout=10,
 155                                              delay=3)
 156         result = ex.execute_unix_command('kubectl describe replicaset {0}'.format(cpu_pooling_pod2['obj_name']))
 157         error = 'Exclusive CPU requests {req} do not match to annotation {proc}'.format(req=request_cpu,
 158                                                                                         proc=annotation_cpu)
 159
 160         if error not in result:
 161             raise Exception('Replicaset description does not contain expected error! -' + result)
 162         else:
 163             logger.info(error)
 164     finally:
 165         common_utils.helm_delete("cpu-pooling")
 166         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 167                                              tester_function=common_utils.test_kubernetes_object_not_available,
 168                                              timeout=90)
 169         cpu_pooling_pod2['obj_type'] = 'pod'
 170
 171
 172 @pabot_lock("flannel_ip")
 173 def step6_more_cpu_annotation_than_request():
 174     annotation_cpu = 2
 175     request_cpu = 1
 176     cpu_pooling_pod2['obj_type'] = 'replicaset'
 177     try:
 178         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive2", release_name="cpu-pooling",
 179                                   values="registry_url={url},nodename={node_name},proc_req={proc},pool_req={req}"
 180                                   .format(url=reg, proc=annotation_cpu, req=request_cpu, node_name=nodename))
 181         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 182                                              tester_function=common_utils.test_kubernetes_object_available,
 183                                              timeout=10,
 184                                              delay=3)
 185         result = ex.execute_unix_command('kubectl describe replicaset {0}'.format(cpu_pooling_pod2['obj_name']))
 186         error = 'Exclusive CPU requests {req} do not match to annotation {proc}'.format(req=request_cpu,
 187                                                                                         proc=annotation_cpu)
 188
 189         if error not in result:
 190             raise Exception('Replicaset description does not contain expected error! -' + result)
 191         else:
 192             logger.info(error)
 193     finally:
 194         common_utils.helm_delete("cpu-pooling")
 195         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 196                                              tester_function=common_utils.test_kubernetes_object_not_available,
 197                                              timeout=90)
 198         cpu_pooling_pod2['obj_type'] = 'pod'
 199
 200
 201 @pabot_lock("flannel_ip")
 202 def step_7_allocate_all_exclusive_and_new_one_start_running_after_needed_resource_is_freed_up():
 203     max_exclusive_pool_size = len(cpupools[nodename]['exclusive_caas'])
 204     try:
 205         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive2", release_name="cpu-pooling1",
 206                                   values="registry_url={reg_url},nodename={node_name},proc_req={cpus},pool_req={cpus}"
 207                                   .format(reg_url=reg, cpus=max_exclusive_pool_size, node_name=nodename))
 208         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod2,
 209                                                     expected_result="1",
 210                                                     filter=r'(Running)\s*[0]',
 211                                                     timeout=90)
 212         logger.info("Allocation of all exclusive CPU successfull!")
 213
 214         common_utils.helm_install(chart_name="default/cpu-pooling-exclusive1", release_name="cpu-pooling2",
 215                                   values="registry_url={reg_url},nodename={node_name}".format(reg_url=reg,
 216                                                                                               node_name=nodename))
 217         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod1,
 218                                                     expected_result="1",
 219                                                     filter=r'(Pending)\s*[0]',
 220                                                     timeout=90,
 221                                                     delay=3)
 222         logger.info("Try to allocate more exclusive CPU -> Pod in Pending!")
 223         common_utils.helm_delete("cpu-pooling1")
 224         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 225                                              tester_function=common_utils.test_kubernetes_object_not_available,
 226                                              timeout=90)
 227         common_utils.test_kubernetes_object_quality(kube_object=cpu_pooling_pod1,
 228                                                     expected_result="1",
 229                                                     filter=r'(Running)\s*[0]',
 230                                                     timeout=90)
 231
 232     finally:
 233         if common_utils.helm_list("cpu-pooling1") != "0":
 234             common_utils.helm_delete("cpu-pooling1")
 235         common_utils.helm_delete("cpu-pooling2")
 236         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod1,
 237                                              tester_function=common_utils.test_kubernetes_object_not_available,
 238                                              timeout=90)
 239         common_utils.check_kubernetes_object(kube_object=cpu_pooling_pod2,
 240                                              tester_function=common_utils.test_kubernetes_object_not_available,
 241                                              timeout=90)
 242
 243
 244 @robot_log
 245 def get_cpu_core_of_process(pod_name, command):
 246     bash_command = "ps | grep '{proc_name}' | grep -v grep | awk '{{print $1}}'".format(proc_name=command)
 247     proc_id = ex.execute_unix_command("kubectl exec `kubectl get pod | grep {0} | "
 248                                       "awk '{{print $1}}'` -- {1}".format(pod_name, bash_command))
 249     bash_command = "cat /proc/{0}/stat | awk '{{print $39}}'".format(proc_id)
 250     result = ex.execute_unix_command("kubectl exec `kubectl get pod | grep {0} | "
 251                                      "awk '{{print $1}}'` -- {1}".format(pod_name, bash_command))
 252     return int(result)