tests/k8s/ha/ha_etcd_api_ctl_sch.robot

   1 ##############################################################################
   2 # Copyright (c) 2019 AT&T Intellectual Property.                             #
   3 # Copyright (c) 2019 Nokia.                                                  #
   4 #                                                                            #
   5 # Licensed under the Apache License, Version 2.0 (the "License");            #
   6 # you maynot use this file except in compliance with the License.            #
   7 #                                                                            #
   8 # You may obtain a copy of the License at                                    #
   9 #       http://www.apache.org/licenses/LICENSE-2.0                           #
  10 #                                                                            #
  11 # Unless required by applicable law or agreed to in writing, software        #
  12 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT  #
  13 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.           #
  14 # See the License for the specific language governing permissions and        #
  15 # limitations under the License.                                             #
  16 ##############################################################################
  17
  18
  19 *** Settings ***
  20 Documentation     HA tests: etcd, api-server, controller-manager, scheduler
  21 Library           SSHLibrary
  22 Library           OperatingSystem
  23 Library           BuiltIn
  24 Suite Setup       Open Connection And Log In
  25 Suite Teardown    Close All Connections
  26
  27 *** Variables ***
  28 ${HOST}           localhost
  29 ${USERNAME}       localadmin
  30 ${NODENAME}       aknode109
  31 ${LOG}            /opt/akraino/validation/ha/print_etcd_api_ctl-manager_sch.txt
  32
  33
  34 ## kubernetes-etcd
  35
  36 *** Test Cases ***
  37 Verify etcd status
  38         [Documentation]         etcd label defined
  39         ${output}=              Execute Command        date
  40         Append To File          ${LOG}  ${output}${\n}
  41         Start Command           kubectl describe node ${NODENAME} | egrep "Labels|etcd"    sudo=True
  42         ${stdout}=              Read Command Output
  43         Append To File          ${LOG}  ${stdout}${\n}
  44         Sleep                   2s
  45         Should Contain          ${stdout}              kubernetes-etcd=enabled
  46
  47
  48 Failure of etcd in node
  49         [Documentation]         kubernetes etcd removed
  50         Start Command           kubectl label node ${NODENAME} kubernetes-etcd-    sudo=True
  51         ${stdout}=              Read Command Output
  52         Append To File          ${LOG}  ${stdout}${\n}
  53         Sleep                   3s
  54         Should Contain          ${stdout}              labeled
  55         Sleep                   40s
  56         ${output}=              Execute Command        kubectl describe node ${NODENAME} | egrep "Labels|etcd"    sudo=True
  57         Append To File          ${LOG}  ${output}${\n}
  58
  59
  60 Enable etcd in node
  61         [Documentation]         etcd label re-established
  62         Start Command           kubectl label node ${NODENAME} kubernetes-etcd=enabled    sudo=True
  63         ${stdout}=              Read Command Output
  64         Append To File          ${LOG}  ${stdout}${\n}
  65         Sleep                   5s
  66         Should Contain          ${stdout}              labeled
  67         Sleep                   40s
  68         ${output}=              Execute Command        kubectl describe node ${NODENAME} | egrep "Labels|etcd"     sudo=True
  69         Append To File          ${LOG}  ${output}${\n}
  70
  71
  72 Verify etcd health
  73         [Documentation]         etcd endpoint healthy
  74         Sleep                   25s
  75         Start Command           kubectl exec -it -n kube-system kubernetes-etcd-${NODENAME} etcdctl endpoint health      sudo=True
  76         ${stdout}=              Read Command Output
  77         Append To File          ${LOG}  ${stdout}${\n}
  78         Sleep                   5s
  79         Should Contain          ${stdout}              is healthy: successfully committed
  80
  81
  82 Check etcd node list
  83         [Documentation]         member list started
  84         Start Command           kubectl exec -it -n kube-system kubernetes-etcd-${NODENAME} etcdctl member list      sudo=True
  85         ${stdout}=              Read Command Output
  86         Append To File          ${LOG}  ${stdout}${\n}
  87         Should Contain          ${stdout}              started, ${NODENAME}
  88
  89 ## kubernetes-api
  90
  91
  92 Verify api-server status
  93         [Documentation]         kubernetes api-server state
  94         Start Command           kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver'     sudo=True
  95         ${stdout}=              Read Command Output
  96         Append To File          ${LOG}  ${stdout}${\n}
  97         Sleep                   2s
  98         Should Contain          ${stdout}              kubernetes-apiserver=enabled
  99
 100
 101 Failure of api-server in node
 102         [Documentation]         kubernetes api-server removed
 103         Start Command           kubectl label node ${NODENAME} kubernetes-apiserver-     sudo=True
 104         ${stdout}=              Read Command Output
 105         Append To File          ${LOG}  ${stdout}${\n}
 106         Sleep                   3s
 107         Should Contain          ${stdout}              labeled
 108         Sleep                   40s
 109         ${output}=              Execute Command        kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver'     sudo=True
 110         Append To File          ${LOG}  ${output}${\n}
 111
 112
 113 Enable api-server in node
 114         [Documentation]         api-serverd label re-established
 115         Start Command           kubectl label node ${NODENAME} kubernetes-apiserver=enabled      sudo=True
 116         ${stdout}=              Read Command Output
 117         Append To File          ${LOG}  ${stdout}${\n}
 118         Sleep                   5s
 119         Should Contain          ${stdout}              labeled
 120         Sleep                   30s
 121
 122
 123 Check api-server status
 124         [Documentation]         kubernetes api-server re-established
 125         Start Command           kubectl describe node ${NODENAME} | egrep 'Label|kubernetes-apiserver'     sudo=True
 126         ${stdout}=              Read Command Output
 127         Append To File          ${LOG}  ${stdout}${\n}
 128         Sleep                   2s
 129         Should Contain          ${stdout}              kubernetes-apiserver=enabled
 130
 131 ##kubernetes controller-manager
 132
 133 Verify controller-manager status
 134         [Documentation]         kubernetes controller-manager state
 135         Start Command           kubectl describe node ${NODENAME} | egrep 'Label|controller-manager'     sudo=True
 136         ${stdout}=              Read Command Output
 137         Append To File          ${LOG}  ${stdout}${\n}
 138         Sleep                   2s
 139         Should Contain          ${stdout}              kubernetes-controller-manager=enabled
 140
 141
 142 Failure of controller-manager in node
 143         [Documentation]         kubernetes controller removed
 144         Start Command           kubectl label node ${NODENAME} kubernetes-controller-manager-     sudo=True
 145         ${stdout}=              Read Command Output
 146         Append To File          ${LOG}  ${stdout}${\n}
 147         Sleep                   3s
 148         Should Contain          ${stdout}              labeled
 149         Sleep                   40s
 150         ${output}=              Execute Command        kubectl describe node ${NODENAME} | egrep 'Label|controller-manager'     sudo=True
 151         Append To File          ${LOG}  ${output}${\n}
 152
 153
 154 Enable controller-manager in node
 155         [Documentation]         controller-manager re-established
 156         Start Command           kubectl label node ${NODENAME} kubernetes-controller-manager=enabled      sudo=True
 157         ${stdout}=              Read Command Output
 158         Append To File          ${LOG}  ${stdout}${\n}
 159         Sleep                   5s
 160         Should Contain          ${stdout}              labeled
 161         Sleep                   30s
 162
 163
 164 Check controller-manager status
 165         [Documentation]         kubernetes controller-manager re-established
 166         Start Command           kubectl describe node ${NODENAME} | egrep 'Label|controller-manager'     sudo=True
 167         ${stdout}=              Read Command Output
 168         Append To File          ${LOG}  ${stdout}${\n}
 169         Sleep                   2s
 170         Should Contain          ${stdout}              kubernetes-controller-manager=enabled
 171
 172 ## kubernetes-scheduler
 173
 174 Verify k8s-scheduler status
 175         [Documentation]         kubernetes scheduler state
 176         Start Command           kubectl describe node ${NODENAME} | egrep 'Label|scheduler'     sudo=True
 177         ${stdout}=              Read Command Output
 178         Append To File          ${LOG}  ${stdout}${\n}
 179         Sleep                   2s
 180         Should Contain          ${stdout}              kubernetes-scheduler=enabled
 181
 182
 183 Failure of scheduler in node
 184         [Documentation]         kubernetes scheduler removed
 185         Start Command           kubectl label node ${NODENAME} kubernetes-scheduler-     sudo=True
 186         ${stdout}=              Read Command Output
 187         Append To File          ${LOG}  ${stdout}${\n}
 188         Sleep                   3s
 189         Should Contain          ${stdout}              labeled
 190         Sleep                   40s
 191         ${output}=              Execute Command        kubectl describe node ${NODENAME} | egrep 'Label|scheduler'     sudo=True
 192         Append To File          ${LOG}  ${output}${\n}
 193
 194
 195 Enable scheduler in node
 196         [Documentation]         scheduler re-established
 197         Start Command           kubectl label node ${NODENAME} kubernetes-scheduler=enabled      sudo=True
 198         ${stdout}=              Read Command Output
 199         Append To File          ${LOG}  ${stdout}${\n}
 200         Sleep                   5s
 201         Should Contain          ${stdout}              labeled
 202         Sleep                   30s
 203
 204
 205 Check k8s scheduler status
 206         [Documentation]         kubernetes scheduler re-established
 207         Start Command           kubectl describe node ${NODENAME} | egrep 'Label|scheduler'     sudo=True
 208         ${stdout}=              Read Command Output
 209         Append To File          ${LOG}  ${stdout}${\n}
 210         Sleep                   2s
 211         Should Contain          ${stdout}              kubernetes-scheduler=enabled
 212
 213
 214 *** Keywords ***
 215 Open Connection And Log In
 216   Open Connection       ${HOST}
 217   Login With Public Key    ${USERNAME}  /root/.ssh/${USERNAME}_id_rsa
 218