testcases/fluentd/tc_001_ssh_test_fluentd_logging.py

   1 import sys
   2 import os
   3 from robot.libraries.BuiltIn import BuiltIn
   4 from robot.api import logger
   5
   6 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '../common'))
   7 import common_utils  # noqa
   8 from decorators_for_robot_functionalities import *  # noqa
   9 from test_constants import *  # noqa
  10
  11
  12 ex = BuiltIn().get_library_instance('execute_command')
  13 stack_infos = BuiltIn().get_library_instance('stack_infos')
  14 log_dir = os.path.join(os.path.dirname(__file__))
  15
  16
  17 def tc_001_ssh_test_fluentd_logging():
  18     steps = ['step1_fluentd_logging_followup_check']
  19     common_utils.keyword_runner(steps)
  20
  21
  22 # Get all pods or list of pods running on a node
  23 def kubernetes_get_all_pods(node, podsList):
  24     pods = []
  25     for po in podsList:
  26         command = "kubectl get po --all-namespaces -o wide | grep " + node + " | grep -vP '"
  27         for pod in pods_skipped[:-1]:
  28             command += pod + '|'
  29         command += pods_skipped[-1] + "' | grep " + po + " | awk '{print $2}'"
  30         stdout = ex.execute_unix_command_on_remote_as_root(command, node)
  31         for line in stdout.splitlines():
  32             pods.append(line)
  33     logger.info(pods)
  34     return pods
  35
  36
  37 # Check wether logs from pods are gathered by fluentd or not
  38 # - Logs from pods in kube-system are monitored
  39 # - Logs from glusterfs in default are monitored
  40 # Looking into fluentd logs for messages like below:
  41 # "2017-10-17 13:03:14 +0000 [info]: plugin/in_tail.rb:586:initialize: following tail of
  42 # /var/log/containers/kube-proxy-172.24.16.104_kube-system_kube-proxy-
  43 # 81ea7d0e0fcfd372ac3cc2a7f980dc7761ede68566b1ef30663cbb1e46307e62.log"
  44 # meaning that e.g. kube-proxy container log is managed by fluentd
  45 # The research starts from the first "fluent starting" message and stops at first "restarting" occurrence if any
  46 def fluentd_logging_followup_check(nodes, followedPods):
  47     for key in nodes:
  48         command = "kubectl get po --all-namespaces -o wide|grep " + nodes[key] + "|grep fluent|awk '{print $2}'"
  49         fluentd = ex.execute_unix_command(command)
  50         pods = kubernetes_get_all_pods(nodes[key], followedPods)
  51         if fluentd is not None:
  52             for pod in pods:
  53                 command = "kubectl -n kube-system logs " + fluentd + \
  54                           "|awk '/starting fluentd/{p=1;next}/restarting/{exit} p'|grep -c 'following.*'" + pod
  55                 logger.info(command)
  56                 stdout = ex.execute_unix_command_on_remote_as_root(command, nodes[key])
  57                 if stdout[0] == '0':
  58                     err = key + ": Pod not followed by fluentd: " + pod
  59                     raise Exception(err)
  60         else:
  61             err = key + ": Fluentd pod not found"
  62             raise Exception(err)
  63
  64
  65 @pabot_lock("cpu_pooling")
  66 def step1_fluentd_logging_followup_check():
  67     logger.console("\nstep1_fluentd_logging_followup_check")
  68
  69     nodes = stack_infos.get_crf_nodes()
  70     # Monitor all pods in kube-system namespace
  71     podsList = ["kube-system"]
  72     fluentd_logging_followup_check(nodes, podsList)
  73
  74     nodes = stack_infos.get_storage_nodes()
  75     # Monitor all pods in kube-system namespace
  76     podsList = ["kube-system"]
  77     fluentd_logging_followup_check(nodes, podsList)
  78
  79     nodes = stack_infos.get_worker_nodes()
  80     # Monitor all pods in kube-system namespace
  81     podsList = ["kube-system"]
  82     fluentd_logging_followup_check(nodes, podsList)