Add tests for Ceph running as systemd service

- monitor/manager failure on one node
- monitor/manager failure on two nodes

JIRA: VAL-19

Change-Id: Ic4fcb8188fc34e9d8e99b53a129724dfc731a54a
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>

diff --git a/ceph/ceph_service.resource b/ceph/ceph_service.resource
new file mode 100644 (file)
index 0000000..048dbcb
--- /dev/null
+++ b/ceph/ceph_service.resource
@@ -0,0 +1,76 @@
+##############################################################################
+# Copyright (c) 2019 AT&T Intellectual Property.                             #
+# Copyright (c) 2019 Nokia.                                                  #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License");            #
+# you maynot use this file except in compliance with the License.            #
+#                                                                            #
+# You may obtain a copy of the License at                                    #
+#       http://www.apache.org/licenses/LICENSE-2.0                           #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT  #
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.           #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+##############################################################################
+
+
+*** Settings ***
+Library           BuiltIn
+Library           SSHLibrary
+Resource          variables.resource
+
+
+*** Keywords ***
+Open Connection And Log In
+    Open Connection          ${HOST}
+    Login With Public Key    ${USERNAME}  ${KEYFILE}
+
+Start Ceph Manager
+    [Arguments]              @{nodes}
+    :FOR  ${node}  IN  @{nodes}
+    \   ${result}=           Run Manager Start  ${node}
+
+Start Ceph Monitor
+    [Arguments]              @{nodes}
+    :FOR  ${node}  IN  @{nodes}
+    \   ${result}=           Run Monitor Start  ${node}
+
+Kill Ceph Manager
+    [Arguments]              @{nodes}
+    :FOR  ${node}  IN  @{nodes}
+    \   ${stdout}=           Execute Command  ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-mgr"
+
+Kill Ceph Monitor
+    [Arguments]              @{nodes}
+    :FOR  ${node}  IN  @{nodes}
+    \   ${stdout}=           Execute Command  ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-mon"
+
+Ceph Should Be Healthy
+    ${stdout}=               Execute Command  ceph health --connect-timeout 20  sudo=True
+    Should Contain  ${stdout}  HEALTH_OK
+
+Ceph Health Should Be Degraded
+    ${stdout}=               Execute Command  ceph health --connect-timeout 20  sudo=True
+    Should Contain  ${stdout}  HEALTH_WARN
+
+Run Manager Start
+    [Arguments]              ${node}
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mgr@${node}"
+    ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl is-active ceph-mgr@${node}"
+    Return From Keyword If   '${stdout}' == 'active'
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl reset-failed ceph-mgr@${node}"
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mgr@${node}"
+    ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl is-active ceph-mgr@${node}"
+    Should Be Equal As Strings  ${stdout}  active
+
+Run Monitor Start
+    [Arguments]              ${node}
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mon@${node}"
+    ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl is-active ceph-mon@${node}"
+    Return From Keyword If   '${stdout}' == 'active'
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl reset-failed ceph-mon@${node}"
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mon@${node}"
+    ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl is-active ceph-mon@${node}"
+    Should Be Equal As Strings  ${stdout}  active

diff --git a/ceph/ceph_service.robot b/ceph/ceph_service.robot
new file mode 100644 (file)
index 0000000..0941b17
--- /dev/null
+++ b/ceph/ceph_service.robot
@@ -0,0 +1,45 @@
+##############################################################################
+# Copyright (c) 2019 AT&T Intellectual Property.                             #
+# Copyright (c) 2019 Nokia.                                                  #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License");            #
+# you maynot use this file except in compliance with the License.            #
+#                                                                            #
+# You may obtain a copy of the License at                                    #
+#       http://www.apache.org/licenses/LICENSE-2.0                           #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT  #
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.           #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+##############################################################################
+
+
+*** Settings ***
+Resource          ceph_service.resource
+Suite Setup       Open Connection And Log In
+Suite Teardown    Close All Connections
+
+
+*** Test Cases ***
+Failure Of Single Monitor And Manager
+    @{nodes}  Create list  ${NODENAME}-1
+    Kill Ceph Monitor  @{nodes}
+    Kill Ceph Manager  @{nodes}
+    Sleep  5s
+    Ceph Health Should Be Degraded
+    Start Ceph Monitor  @{nodes}
+    Start Ceph Manager  @{nodes}
+    Sleep  10s
+    Ceph Should Be Healthy
+
+Failure Of Two Monitors And Managers
+    @{nodes}  Create list  ${NODENAME}-1  ${NODENAME}-2
+    Kill Ceph Monitor  @{nodes}
+    Kill Ceph Manager  @{nodes}
+    Sleep  5s
+    Start Ceph Monitor  @{nodes}
+    Start Ceph Manager  @{nodes}
+    Sleep  10s
+    Ceph Should Be Healthy

diff --git a/ceph/variables.resource b/ceph/variables.resource
new file mode 100644 (file)
index 0000000..fb6e3e5
--- /dev/null
+++ b/ceph/variables.resource
@@ -0,0 +1,24 @@
+##############################################################################
+# Copyright (c) 2019 AT&T Intellectual Property.                             #
+# Copyright (c) 2019 Nokia.                                                  #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License");            #
+# you maynot use this file except in compliance with the License.            #
+#                                                                            #
+# You may obtain a copy of the License at                                    #
+#       http://www.apache.org/licenses/LICENSE-2.0                           #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT  #
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.           #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+##############################################################################
+
+
+*** Variables ***
+${HOST}           localhost                          # cluster's master node address
+${USERNAME}       cloudadmin                         # user to log in to the host
+${KEYFILE}        ~/.ssh/id_rsa                      # identity file for authentication
+${NODENAME}       recmaster                          # cluster's master node base name
+${SSH_CMD}        ssh -o StrictHostKeyChecking=no    # disable host key checking