1 ##############################################################################
2 # Copyright (c) 2019 AT&T Intellectual Property. #
3 # Copyright (c) 2019 Nokia. #
5 # Licensed under the Apache License, Version 2.0 (the "License"); #
6 # you maynot use this file except in compliance with the License. #
8 # You may obtain a copy of the License at #
9 # http://www.apache.org/licenses/LICENSE-2.0 #
11 # Unless required by applicable law or agreed to in writing, software #
12 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT #
13 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
14 # See the License for the specific language governing permissions and #
15 # limitations under the License. #
16 ##############################################################################
20 Documentation HA tests: etcd, api-server, controller-manager, scheduler
22 Library OperatingSystem
24 Suite Setup Open Connection And Log In
25 Suite Teardown Close All Connections
29 ${USERNAME} localadmin
31 ${LOG} ${LOG_PATH}${/}${SUITE_NAME.replace(' ','_')}.log
38 [Documentation] etcd label defined
39 ${output}= Execute Command date
40 Append To File ${LOG} ${output}${\n}
41 Start Command kubectl describe node ${NODENAME} | egrep "Labels|etcd" sudo=True
42 ${stdout}= Read Command Output
43 Append To File ${LOG} ${stdout}${\n}
45 Should Contain ${stdout} kubernetes-etcd=enabled
48 Failure of etcd in node
49 [Documentation] kubernetes etcd removed
50 Start Command kubectl label node ${NODENAME} kubernetes-etcd- sudo=True
51 ${stdout}= Read Command Output
52 Append To File ${LOG} ${stdout}${\n}
54 Should Contain ${stdout} labeled
56 ${output}= Execute Command kubectl describe node ${NODENAME} | egrep "Labels|etcd" sudo=True
57 Append To File ${LOG} ${output}${\n}
61 [Documentation] etcd label re-established
62 Start Command kubectl label node ${NODENAME} kubernetes-etcd=enabled sudo=True
63 ${stdout}= Read Command Output
64 Append To File ${LOG} ${stdout}${\n}
66 Should Contain ${stdout} labeled
68 ${output}= Execute Command kubectl describe node ${NODENAME} | egrep "Labels|etcd" sudo=True
69 Append To File ${LOG} ${output}${\n}
73 [Documentation] etcd endpoint healthy
75 Start Command kubectl exec -it -n kube-system kubernetes-etcd-${NODENAME} etcdctl endpoint health sudo=True
76 ${stdout}= Read Command Output
77 Append To File ${LOG} ${stdout}${\n}
79 Should Contain ${stdout} is healthy: successfully committed
83 [Documentation] member list started
84 Start Command kubectl exec -it -n kube-system kubernetes-etcd-${NODENAME} etcdctl member list sudo=True
85 ${stdout}= Read Command Output
86 Append To File ${LOG} ${stdout}${\n}
87 Should Contain ${stdout} started, ${NODENAME}
92 Verify api-server status
93 [Documentation] kubernetes api-server state
94 Start Command kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver' sudo=True
95 ${stdout}= Read Command Output
96 Append To File ${LOG} ${stdout}${\n}
98 Should Contain ${stdout} kubernetes-apiserver=enabled
101 Failure of api-server in node
102 [Documentation] kubernetes api-server removed
103 Start Command kubectl label node ${NODENAME} kubernetes-apiserver- sudo=True
104 ${stdout}= Read Command Output
105 Append To File ${LOG} ${stdout}${\n}
107 Should Contain ${stdout} labeled
109 ${output}= Execute Command kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver' sudo=True
110 Append To File ${LOG} ${output}${\n}
113 Enable api-server in node
114 [Documentation] api-serverd label re-established
115 Start Command kubectl label node ${NODENAME} kubernetes-apiserver=enabled sudo=True
116 ${stdout}= Read Command Output
117 Append To File ${LOG} ${stdout}${\n}
119 Should Contain ${stdout} labeled
123 Check api-server status
124 [Documentation] kubernetes api-server re-established
125 Start Command kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver' sudo=True
126 ${stdout}= Read Command Output
127 Append To File ${LOG} ${stdout}${\n}
129 Should Contain ${stdout} kubernetes-apiserver=enabled
131 ##kubernetes controller-manager
133 Verify controller-manager status
134 [Documentation] kubernetes controller-manager state
135 Start Command kubectl describe node ${NODENAME} | egrep 'Label|controller-manager' sudo=True
136 ${stdout}= Read Command Output
137 Append To File ${LOG} ${stdout}${\n}
139 Should Contain ${stdout} kubernetes-controller-manager=enabled
142 Failure of controller-manager in node
143 [Documentation] kubernetes controller removed
144 Start Command kubectl label node ${NODENAME} kubernetes-controller-manager- sudo=True
145 ${stdout}= Read Command Output
146 Append To File ${LOG} ${stdout}${\n}
148 Should Contain ${stdout} labeled
150 ${output}= Execute Command kubectl describe node ${NODENAME} | egrep 'Label|controller-manager' sudo=True
151 Append To File ${LOG} ${output}${\n}
154 Enable controller-manager in node
155 [Documentation] controller-manager re-established
156 Start Command kubectl label node ${NODENAME} kubernetes-controller-manager=enabled sudo=True
157 ${stdout}= Read Command Output
158 Append To File ${LOG} ${stdout}${\n}
160 Should Contain ${stdout} labeled
164 Check controller-manager status
165 [Documentation] kubernetes controller-manager re-established
166 Start Command kubectl describe node ${NODENAME} | egrep 'Label|controller-manager' sudo=True
167 ${stdout}= Read Command Output
168 Append To File ${LOG} ${stdout}${\n}
170 Should Contain ${stdout} kubernetes-controller-manager=enabled
172 ## kubernetes-scheduler
174 Verify k8s-scheduler status
175 [Documentation] kubernetes scheduler state
176 Start Command kubectl describe node ${NODENAME} | egrep 'Label|scheduler' sudo=True
177 ${stdout}= Read Command Output
178 Append To File ${LOG} ${stdout}${\n}
180 Should Contain ${stdout} kubernetes-scheduler=enabled
183 Failure of scheduler in node
184 [Documentation] kubernetes scheduler removed
185 Start Command kubectl label node ${NODENAME} kubernetes-scheduler- sudo=True
186 ${stdout}= Read Command Output
187 Append To File ${LOG} ${stdout}${\n}
189 Should Contain ${stdout} labeled
191 ${output}= Execute Command kubectl describe node ${NODENAME} | egrep 'Label|scheduler' sudo=True
192 Append To File ${LOG} ${output}${\n}
195 Enable scheduler in node
196 [Documentation] scheduler re-established
197 Start Command kubectl label node ${NODENAME} kubernetes-scheduler=enabled sudo=True
198 ${stdout}= Read Command Output
199 Append To File ${LOG} ${stdout}${\n}
201 Should Contain ${stdout} labeled
205 Check k8s scheduler status
206 [Documentation] kubernetes scheduler re-established
207 Start Command kubectl describe node ${NODENAME} | egrep 'Label|scheduler' sudo=True
208 ${stdout}= Read Command Output
209 Append To File ${LOG} ${stdout}${\n}
211 Should Contain ${stdout} kubernetes-scheduler=enabled
215 Open Connection And Log In
216 Open Connection ${HOST}
217 Login With Public Key ${USERNAME} /root/.ssh/${USERNAME}_id_rsa