import sys import os from robot.libraries.BuiltIn import BuiltIn from robot.api import logger sys.path.append(os.path.join(os.path.dirname(__file__), '..', '../common')) import common_utils # noqa from decorators_for_robot_functionalities import * # noqa from test_constants import * # noqa ex = BuiltIn().get_library_instance('execute_command') stack_infos = BuiltIn().get_library_instance('stack_infos') log_dir = os.path.join(os.path.dirname(__file__)) def tc_001_ssh_test_fluentd_logging(): steps = ['step1_fluentd_logging_followup_check'] common_utils.keyword_runner(steps) # Get all pods or list of pods running on a node def kubernetes_get_all_pods(node, podsList): pods = [] for po in podsList: command = "kubectl get po --all-namespaces -o wide | grep " + node + " | grep -vP '" for pod in pods_skipped[:-1]: command += pod + '|' command += pods_skipped[-1] + "' | grep " + po + " | awk '{print $2}'" stdout = ex.execute_unix_command_on_remote_as_root(command, node) for line in stdout.splitlines(): pods.append(line) logger.info(pods) return pods # Check wether logs from pods are gathered by fluentd or not # - Logs from pods in kube-system are monitored # - Logs from glusterfs in default are monitored # Looking into fluentd logs for messages like below: # "2017-10-17 13:03:14 +0000 [info]: plugin/in_tail.rb:586:initialize: following tail of # /var/log/containers/kube-proxy-172.24.16.104_kube-system_kube-proxy- # 81ea7d0e0fcfd372ac3cc2a7f980dc7761ede68566b1ef30663cbb1e46307e62.log" # meaning that e.g. kube-proxy container log is managed by fluentd # The research starts from the first "fluent starting" message and stops at first "restarting" occurrence if any def fluentd_logging_followup_check(nodes, followedPods): for key in nodes: command = "kubectl get po --all-namespaces -o wide|grep " + nodes[key] + "|grep fluent|awk '{print $2}'" fluentd = ex.execute_unix_command(command) pods = kubernetes_get_all_pods(nodes[key], followedPods) if fluentd is not None: for pod in pods: command = "kubectl -n kube-system logs " + fluentd + \ "|awk '/starting fluentd/{p=1;next}/restarting/{exit} p'|grep -c 'following.*'" + pod logger.info(command) stdout = ex.execute_unix_command_on_remote_as_root(command, nodes[key]) if stdout[0] == '0': err = key + ": Pod not followed by fluentd: " + pod raise Exception(err) else: err = key + ": Fluentd pod not found" raise Exception(err) @pabot_lock("cpu_pooling") def step1_fluentd_logging_followup_check(): logger.console("\nstep1_fluentd_logging_followup_check") nodes = stack_infos.get_crf_nodes() # Monitor all pods in kube-system namespace podsList = ["kube-system"] fluentd_logging_followup_check(nodes, podsList) nodes = stack_infos.get_storage_nodes() # Monitor all pods in kube-system namespace podsList = ["kube-system"] fluentd_logging_followup_check(nodes, podsList) nodes = stack_infos.get_worker_nodes() # Monitor all pods in kube-system namespace podsList = ["kube-system"] fluentd_logging_followup_check(nodes, podsList)