From 3c89b99ccaa8abef3a1e376a63b46de3d70af8b4 Mon Sep 17 00:00:00 2001 From: Miguel Montiel Date: Tue, 2 Apr 2019 21:46:52 +0000 Subject: [PATCH] Set of TCs to test K8s layer via Robot Change-Id: I33986759b7904978ac74c033d7ba02f38441d1c3 --- bios_version/bios_version_dell.robot | 63 ++++++++++ conformance/conformance.robot | 75 ++++++++++++ conformance/sonobuoy.yaml | 196 +++++++++++++++++++++++++++++++ e2e/e2e.sh | 30 +++++ ha/ha_calico_dns_proxy.robot | 118 +++++++++++++++++++ ha/ha_etcd_api_ctl_sch.robot | 218 +++++++++++++++++++++++++++++++++++ ha/ha_services.robot | 108 +++++++++++++++++ ha/ha_worker.robot | 50 ++++++++ 8 files changed, 858 insertions(+) create mode 100644 bios_version/bios_version_dell.robot create mode 100644 conformance/conformance.robot create mode 100644 conformance/sonobuoy.yaml create mode 100644 e2e/e2e.sh create mode 100644 ha/ha_calico_dns_proxy.robot create mode 100644 ha/ha_etcd_api_ctl_sch.robot create mode 100644 ha/ha_services.robot create mode 100644 ha/ha_worker.robot diff --git a/bios_version/bios_version_dell.robot b/bios_version/bios_version_dell.robot new file mode 100644 index 0000000..efa9dd9 --- /dev/null +++ b/bios_version/bios_version_dell.robot @@ -0,0 +1,63 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Documentation Check BIOS and HW details +Library SSHLibrary +Library OperatingSystem +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections + +*** Variables *** +${HOST} localhost +${USERNAME} localadmin +${LOG} /opt/akraino/validation/bios_version/print_bios.txt +${SYSINFO} PowerEdge R740xd + +*** Test Cases *** +Get HW Details + [Documentation] Verify HW details + Start Command dmidecode | grep -A3 '^System Information' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} ${SYSINFO} + +Verify BIOS Revision + [Documentation] Verify BIOS Revision + Start Command dmidecode | more | grep 'BIOS Revision' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} BIOS Revision: 1.3 + +Check NUMA and CPU + [Documentation] NUMAs and CPU components + ${output}= Execute Command lscpu + Append To File ${LOG} ${output}${\n} + Should Contain ${output} CPU(s): 88 + +Verify Block Devices + [Documentation] Reads the sysfs filesystem + ${output}= Execute Command lsblk + Append To File ${LOG} ${output}${\n} + Should Contain ${output} sdg4 + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} /root/.ssh/${USERNAME}_id_rsa + diff --git a/conformance/conformance.robot b/conformance/conformance.robot new file mode 100644 index 0000000..1e0e636 --- /dev/null +++ b/conformance/conformance.robot @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Documentation Run K8s Conformance Test +Library SSHLibrary +Library OperatingSystem +Library BuiltIn +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections + +*** Variables *** +${HOST} localhost +${USERNAME} localadmin +${LOG} /opt/akraino/validation/conformance/print_conformance.txt + +*** Test Cases *** +Get Robot Version + [Documentation] Verify Robot + Start Command robot --version + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Robot Framework + +Verify Go Package + [Documentation] Verify Go Package + Start Command hello + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Hello, world! + + +Start Sonobuoy Conformance Test + [Documentation] Test will take about 1hr and 40 mins to complete + ${output}= Execute Command cat /opt/akraino/validation/conformance/sonobuoy.yaml | kubectl apply -f - sudo=True + Append To File ${LOG} ${output}${\n} + Sleep 3s + Should Contain ${output} pod/sonobuoy created + + +Description of Sonobuoy Pods + [Documentation] Description of Sonobuoy Pod + ${output} ${rc}= Execute Command kubectl describe pod/sonobuoy -n heptio-sonobuoy return_rc=True sudo=True + Append To File ${LOG} ${output}${\n} + Should Be Equal As Integers ${rc} 0 + + +Verify Conformance Test is Running + [Documentation] Conformance Test ongoing + ${output} ${rc}= Execute Command kubectl get pods --all-namespaces -o wide | grep heptio return_rc=True sudo=True + Append To File ${LOG} ${output}${\n} + Should Be Equal As Integers ${rc} 0 + + + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} /root/.ssh/${USERNAME}_id_rsa + diff --git a/conformance/sonobuoy.yaml b/conformance/sonobuoy.yaml new file mode 100644 index 0000000..89d9385 --- /dev/null +++ b/conformance/sonobuoy.yaml @@ -0,0 +1,196 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: heptio-sonobuoy +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount + namespace: heptio-sonobuoy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount-heptio-sonobuoy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sonobuoy-serviceaccount +subjects: +- kind: ServiceAccount + name: sonobuoy-serviceaccount + namespace: heptio-sonobuoy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - '/metrics' + - '/logs' + - '/logs/*' + verbs: + - 'get' +--- +apiVersion: v1 +data: + config.json: | + {"Description":"DEFAULT","UUID":"376a4ddc-4498-49fc-af2e-999242c4c245","Version":"v0.13.0","ResultsDir":"/tmp/sonobuoy","Resources":["CertificateSigningRequests","ClusterRoleBindings","ClusterRoles","ComponentStatuses","CustomResourceDefinitions","Nodes","PersistentVolumes","PodSecurityPolicies","ServerGroups","ServerVersion","StorageClasses","ConfigMaps","ControllerRevisions","CronJobs","DaemonSets","Deployments","Endpoints","Ingresses","Jobs","LimitRanges","NetworkPolicies","PersistentVolumeClaims","PodDisruptionBudgets","PodTemplates","Pods","ReplicaSets","ReplicationControllers","ResourceQuotas","RoleBindings","Roles","ServiceAccounts","Services","StatefulSets"],"Filters":{"Namespaces":".*","LabelSelector":""},"Limits":{"PodLogs":{"LimitSize":"","LimitTime":""}},"Server":{"bindaddress":"0.0.0.0","bindport":8080,"advertiseaddress":"","timeoutseconds":10800},"Plugins":[{"name":"e2e"},{"name":"systemd-logs"}],"PluginSearchPath":["./plugins.d","/etc/sonobuoy/plugins.d","~/sonobuoy/plugins.d"],"Namespace":"heptio-sonobuoy","LoadedPlugins":null,"WorkerImage":"gcr.io/heptio-images/sonobuoy:latest","ImagePullPolicy":"Always"} +kind: ConfigMap +metadata: + labels: + component: sonobuoy + name: sonobuoy-config-cm + namespace: heptio-sonobuoy +--- +apiVersion: v1 +data: + e2e.yaml: | + sonobuoy-config: + driver: Job + plugin-name: e2e + result-type: e2e + spec: + env: + - name: E2E_FOCUS + value: '\[Conformance\]' + - name: E2E_SKIP + value: 'Alpha|\[(Disruptive|Feature:[^\]]+|Flaky)\]' + - name: E2E_PARALLEL + value: '1' + command: ["/run_e2e.sh"] + image: gcr.io/heptio-images/kube-conformance:v1.11 + imagePullPolicy: Always + name: e2e + volumeMounts: + - mountPath: /tmp/results + name: results + readOnly: false + tolerations: + - operator: "Exists" + systemd-logs.yaml: | + sonobuoy-config: + driver: DaemonSet + plugin-name: systemd-logs + result-type: systemd_logs + spec: + command: ["/bin/sh", "-c", "/get_systemd_logs.sh && sleep 3600"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: RESULTS_DIR + value: /tmp/results + - name: CHROOT_DIR + value: /node + image: gcr.io/heptio-images/sonobuoy-plugin-systemd-logs:latest + imagePullPolicy: Always + name: sonobuoy-systemd-logs-config + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp/results + name: results + readOnly: false + - mountPath: /node + name: root + readOnly: false +kind: ConfigMap +metadata: + labels: + component: sonobuoy + name: sonobuoy-plugins-cm + namespace: heptio-sonobuoy +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + component: sonobuoy + run: sonobuoy-master + tier: analysis + name: sonobuoy + namespace: heptio-sonobuoy +spec: + containers: + - command: + - /bin/bash + - -c + - /sonobuoy master --no-exit=true -v 3 --logtostderr + env: + - name: SONOBUOY_ADVERTISE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: gcr.io/heptio-images/sonobuoy:v0.13.0 + imagePullPolicy: Always + name: kube-sonobuoy + volumeMounts: + - mountPath: /etc/sonobuoy + name: sonobuoy-config-volume + - mountPath: /plugins.d + name: sonobuoy-plugins-volume + - mountPath: /tmp/sonobuoy + name: output-volume + restartPolicy: Never + serviceAccountName: sonobuoy-serviceaccount + volumes: + - configMap: + name: sonobuoy-config-cm + name: sonobuoy-config-volume + - configMap: + name: sonobuoy-plugins-cm + name: sonobuoy-plugins-volume + - emptyDir: {} + name: output-volume +--- +apiVersion: v1 +kind: Service +metadata: + labels: + component: sonobuoy + run: sonobuoy-master + name: sonobuoy-master + namespace: heptio-sonobuoy +spec: + ports: + - port: 8080 + protocol: TCP + targetPort: 8080 + selector: + run: sonobuoy-master + type: ClusterIP + diff --git a/e2e/e2e.sh b/e2e/e2e.sh new file mode 100644 index 0000000..f1b9284 --- /dev/null +++ b/e2e/e2e.sh @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + +#!/bin/bash +# +echo "`date +%H:%M:%S` : Starting Kubetest build" +cd /root/go/src/k8s.io/kubernetes +pwd +kubetest --build +echo "Build completed" +export KUBECONFIG=/etc/kubernetes/config +kubectl proxy --port=8080 & +echo "`date +%H:%M:%S` : Start Kubetest" +kubetest --provider=local --test --test_args="--minStartupPods=3 --ginkgo.focus=Secrets" --check-version-skew=false +echo "`date +%H:%M:%S` : Kubetest finished" + diff --git a/ha/ha_calico_dns_proxy.robot b/ha/ha_calico_dns_proxy.robot new file mode 100644 index 0000000..7f2f2ab --- /dev/null +++ b/ha/ha_calico_dns_proxy.robot @@ -0,0 +1,118 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Documentation HA test cases for calico, coredns and haproxy +Library SSHLibrary +Library OperatingSystem +Library BuiltIn +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections + +*** Variables *** +${HOST} localhost +${USERNAME} localadmin +${LOG} /opt/akraino/validation/ha/print_calico_coredns_haproxy.txt + + + +*** Test Cases *** + +## Calico + +Verify calico status + [Documentation] Calico nodes are active + Start Command kubectl get pod -n kube-system -o wide -l k8s-app=calico-node sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Running + +Failure of calico nodes + [Documentation] Calico pods deleted + ${output}= Execute Command kubectl delete pod -n kube-system -l k8s-app=calico-node sudo=True + Append To File ${LOG} ${output}${\n} + Sleep 5s + Should Contain ${output} deleted + +Verify calico node re-establishment + [Documentation] Calico component is auto-created and running + Sleep 5s + Start Command kubectl get pod -n kube-system -o wide -l k8s-app=calico-node sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Running + +## Coredns + +Verify coredns status + [Documentation] Coredns components active + Start Command kubectl get pod -n kube-system -o wide -l coredns=enabled sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Running + + +Failure of coredns in control plane node + [Documentation] coredns pod is deleted + ${output}= Execute Command kubectl delete pod -n kube-system -l coredns=enabled sudo=True + Append To File ${LOG} ${output}${\n} + Sleep 30s + Should Contain ${output} deleted + + +Verify coredns re-establishment + [Documentation] Coredns component is auto-created and running + Sleep 5s + Start Command kubectl get pod -n kube-system -l coredns=enabled sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Running + +## Haproxy + +Verify haproxy status + [Documentation] All haproxy pods are active + Start Command kubectl get pod -n kube-system -o wide -l application=haproxy sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Running + + +Failure of haproxy + [Documentation] Haproxy components deleted + ${output}= Execute Command kubectl delete pod -n kube-system -l application=haproxy sudo=True + Append To File ${LOG} ${output}${\n} + Sleep 5s + Should Contain ${output} deleted + + +Verify haproxy re-establishment + [Documentation] Haproxy components auto-created and running + Sleep 20s + Start Command kubectl get pod -n kube-system -o wide -l application=haproxy sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} Running + + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} /root/.ssh/${USERNAME}_id_rsa + + diff --git a/ha/ha_etcd_api_ctl_sch.robot b/ha/ha_etcd_api_ctl_sch.robot new file mode 100644 index 0000000..31b6f0b --- /dev/null +++ b/ha/ha_etcd_api_ctl_sch.robot @@ -0,0 +1,218 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Documentation HA tests: etcd, api-server, controller-manager, scheduler +Library SSHLibrary +Library OperatingSystem +Library BuiltIn +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections + +*** Variables *** +${HOST} localhost +${USERNAME} localadmin +${NODENAME} aknode109 +${LOG} /opt/akraino/validation/ha/print_etcd_api_ctl-manager_sch.txt + + +## kubernetes-etcd + +*** Test Cases *** +Verify etcd status + [Documentation] etcd label defined + ${output}= Execute Command date + Append To File ${LOG} ${output}${\n} + Start Command kubectl describe node ${NODENAME} | egrep "Labels|etcd" sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-etcd=enabled + + +Failure of etcd in node + [Documentation] kubernetes etcd removed + Start Command kubectl label node ${NODENAME} kubernetes-etcd- sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 3s + Should Contain ${stdout} labeled + Sleep 40s + ${output}= Execute Command kubectl describe node ${NODENAME} | egrep "Labels|etcd" sudo=True + Append To File ${LOG} ${output}${\n} + + +Enable etcd in node + [Documentation] etcd label re-established + Start Command kubectl label node ${NODENAME} kubernetes-etcd=enabled sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 5s + Should Contain ${stdout} labeled + Sleep 40s + ${output}= Execute Command kubectl describe node ${NODENAME} | egrep "Labels|etcd" sudo=True + Append To File ${LOG} ${output}${\n} + + +Verify etcd health + [Documentation] etcd endpoint healthy + Sleep 25s + Start Command kubectl exec -it -n kube-system kubernetes-etcd-${NODENAME} etcdctl endpoint health sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 5s + Should Contain ${stdout} is healthy: successfully committed + + +Check etcd node list + [Documentation] member list started + Start Command kubectl exec -it -n kube-system kubernetes-etcd-${NODENAME} etcdctl member list sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Contain ${stdout} started, ${NODENAME} + +## kubernetes-api + + +Verify api-server status + [Documentation] kubernetes api-server state + Start Command kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-apiserver=enabled + + +Failure of api-server in node + [Documentation] kubernetes api-server removed + Start Command kubectl label node ${NODENAME} kubernetes-apiserver- sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 3s + Should Contain ${stdout} labeled + Sleep 40s + ${output}= Execute Command kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver' sudo=True + Append To File ${LOG} ${output}${\n} + + +Enable api-server in node + [Documentation] api-serverd label re-established + Start Command kubectl label node ${NODENAME} kubernetes-apiserver=enabled sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 5s + Should Contain ${stdout} labeled + Sleep 30s + + +Check api-server status + [Documentation] kubernetes api-server re-established + Start Command kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-apiserver=enabled + +##kubernetes controller-manager + +Verify controller-manager status + [Documentation] kubernetes controller-manager state + Start Command kubectl describe node ${NODENAME} | egrep 'Label|controller-manager' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-controller-manager=enabled + + +Failure of controller-manager in node + [Documentation] kubernetes controller removed + Start Command kubectl label node ${NODENAME} kubernetes-controller-manager- sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 3s + Should Contain ${stdout} labeled + Sleep 40s + ${output}= Execute Command kubectl describe node ${NODENAME} | egrep 'Label|controller-manager' sudo=True + Append To File ${LOG} ${output}${\n} + + +Enable controller-manager in node + [Documentation] controller-manager re-established + Start Command kubectl label node ${NODENAME} kubernetes-controller-manager=enabled sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 5s + Should Contain ${stdout} labeled + Sleep 30s + + +Check controller-manager status + [Documentation] kubernetes controller-manager re-established + Start Command kubectl describe node ${NODENAME} | egrep 'Label|controller-manager' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-controller-manager=enabled + +## kubernetes-scheduler + +Verify k8s-scheduler status + [Documentation] kubernetes scheduler state + Start Command kubectl describe node ${NODENAME} | egrep 'Label|scheduler' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-scheduler=enabled + + +Failure of scheduler in node + [Documentation] kubernetes scheduler removed + Start Command kubectl label node ${NODENAME} kubernetes-scheduler- sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 3s + Should Contain ${stdout} labeled + Sleep 40s + ${output}= Execute Command kubectl describe node ${NODENAME} | egrep 'Label|scheduler' sudo=True + Append To File ${LOG} ${output}${\n} + + +Enable scheduler in node + [Documentation] scheduler re-established + Start Command kubectl label node ${NODENAME} kubernetes-scheduler=enabled sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 5s + Should Contain ${stdout} labeled + Sleep 30s + + +Check k8s scheduler status + [Documentation] kubernetes scheduler re-established + Start Command kubectl describe node ${NODENAME} | egrep 'Label|scheduler' sudo=True + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Sleep 2s + Should Contain ${stdout} kubernetes-scheduler=enabled + + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} /root/.ssh/${USERNAME}_id_rsa + diff --git a/ha/ha_services.robot b/ha/ha_services.robot new file mode 100644 index 0000000..3e1e7be --- /dev/null +++ b/ha/ha_services.robot @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Documentation HA services tests: docker and kubelet +Library SSHLibrary +Library OperatingSystem +Library BuiltIn +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections + +*** Variables *** +${HOST} localhost +${USERNAME} localadmin +${LOG} /opt/akraino/validation/ha/print_docker_kubelet.txt + + +## Container Runtime + +*** Test Cases *** +Verify status of container runtime + [Documentation] container runtime active + Start Command systemctl is-active docker.service + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Be Equal ${stdout} active + + +Fail container runtime + [Documentation] container runtime stopped + ${output} ${rc}= Execute Command systemctl stop docker.service return_rc=True sudo=True + Append To File ${LOG} ${output}${\n} + Should Be Equal As Integers ${rc} 0 + Sleep 8s + Start Command systemctl is-active docker.service + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Be Equal ${stdout} inactive + + +Start container runtime + [Documentation] container runtime active + Sleep 1 minute + ${output} ${rc}= Execute Command systemctl start docker.service return_rc=True sudo=True + Append To File ${LOG} ${output}${\n} + Should Be Equal As Integers ${rc} 0 + Sleep 8s + Start Command systemctl is-active docker.service + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Be Equal ${stdout} active + +## Kubelet Service + +Verify kubelet status + [Documentation] kubelet service active + Start Command systemctl is-active kubelet.service + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Be Equal ${stdout} active + + +Fail kubelet service + [Documentation] kubelet service stopped + ${output} ${rc}= Execute Command systemctl stop kubelet.service return_rc=True sudo=True + Append To File ${LOG} ${output}${\n} + Should Be Equal As Integers ${rc} 0 + Sleep 8s + Start Command systemctl is-active kubelet.service + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Be Equal ${stdout} inactive + + +Start kubelet service + [Documentation] kubelet active + Sleep 30s + ${output} ${rc}= Execute Command systemctl start kubelet.service return_rc=True sudo=True + Append To File ${LOG} ${output}${\n} + Should Be Equal As Integers ${rc} 0 + Sleep 8s + Start Command systemctl is-active kubelet.service + ${stdout}= Read Command Output + Append To File ${LOG} ${stdout}${\n} + Should Be Equal ${stdout} active + + + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} /root/.ssh/${USERNAME}_id_rsa + diff --git a/ha/ha_worker.robot b/ha/ha_worker.robot new file mode 100644 index 0000000..3019edf --- /dev/null +++ b/ha/ha_worker.robot @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + +*** Settings *** +Documentation Run HA Test - Fail Control Plane 3 +Library SSHLibrary +Library OperatingSystem +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections + +*** Variables *** +${HOST} localhost +${USERNAME} ipmi_admin_username +${PASSWORD} ipmi_admin_password +${LOG} /opt/akraino/validation/ha/print_ha.txt + +*** Test Cases *** +Power Status of Host + [Documentation] Get Power Status on iDRAC + ${output}= Execute Command racadm serveraction powerstatus + Append To File ${LOG} ${output}${\n} + Should Contain ${output} ON + + +Fail Control Plane + [Documentation] Perform a reboot operation via iDRAC + ${output} Execute Command racadm serveraction hardreset + Append To File ${LOG} ${output}${\n} + Should Contain ${output} successful + + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login ${USERNAME} ${PASSWORD} + -- 2.16.6