Add tests for Ceph OSD running as systemd service

- Ceph services are available when Ceph OSD is failing
- Ceph OSD recovers back to operational

JIRA: VAL-22

Change-Id: Id7b7324df5437cfeecc103dacb165b281e26cb0e
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>

diff --git a/ceph/ceph_service.resource b/ceph/ceph_service.resource
index 048dbcb..62603f2 100644 (file)
--- a/ceph/ceph_service.resource
+++ b/ceph/ceph_service.resource
@@ -37,6 +37,11 @@ Start Ceph Monitor
     :FOR  ${node}  IN  @{nodes}
     \   ${result}=           Run Monitor Start  ${node}
 
     :FOR  ${node}  IN  @{nodes}
     \   ${result}=           Run Monitor Start  ${node}
 

+Start Ceph OSD
+    [Arguments]              @{nodes}
+    :FOR  ${node}  IN  @{nodes}
+    \   ${result}=           Run Ceph OSD Start  ${node}
+

 Kill Ceph Manager
     [Arguments]              @{nodes}
     :FOR  ${node}  IN  @{nodes}
 Kill Ceph Manager
     [Arguments]              @{nodes}
     :FOR  ${node}  IN  @{nodes}


@@ -47,6 +52,11 @@ Kill Ceph Monitor
     :FOR  ${node}  IN  @{nodes}
     \   ${stdout}=           Execute Command  ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-mon"
 
     :FOR  ${node}  IN  @{nodes}
     \   ${stdout}=           Execute Command  ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-mon"
 

+Kill Ceph OSD
+    [Arguments]              @{nodes}
+    :FOR  ${node}  IN  @{nodes}
+    \   ${stdout}=           Execute Command  ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-osd"
+

 Ceph Should Be Healthy
     ${stdout}=               Execute Command  ceph health --connect-timeout 20  sudo=True
     Should Contain  ${stdout}  HEALTH_OK
 Ceph Should Be Healthy
     ${stdout}=               Execute Command  ceph health --connect-timeout 20  sudo=True
     Should Contain  ${stdout}  HEALTH_OK


@@ -55,6 +65,16 @@ Ceph Health Should Be Degraded
     ${stdout}=               Execute Command  ceph health --connect-timeout 20  sudo=True
     Should Contain  ${stdout}  HEALTH_WARN
 
     ${stdout}=               Execute Command  ceph health --connect-timeout 20  sudo=True
     Should Contain  ${stdout}  HEALTH_WARN
 

+Number Of OSDs Up
+    ${stdout}=               Execute Command  ceph -f json osd stat  sudo=True
+    ${stat}=                 Evaluate  json.loads($stdout)  json
+    [Return]                 ${stat["num_up_osds"]}
+
+Number Of OSDs Up Should Be
+    [Arguments]              ${num_expected}
+    ${num_up_osds}=          Number Of OSDs Up
+    Should Be Equal          ${num_up_osds}  ${num_expected}
+

 Run Manager Start
     [Arguments]              ${node}
     Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mgr@${node}"
 Run Manager Start
     [Arguments]              ${node}
     Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mgr@${node}"


@@ -74,3 +94,13 @@ Run Monitor Start
     Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mon@${node}"
     ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl is-active ceph-mon@${node}"
     Should Be Equal As Strings  ${stdout}  active
     Execute Command          ${SSH_CMD} ${node} "sudo systemctl start ceph-mon@${node}"
     ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl is-active ceph-mon@${node}"
     Should Be Equal As Strings  ${stdout}  active

+
+Run Ceph OSD Start
+    [Arguments]              ${node}
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl -a start ceph-osd@*"
+    ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl -a is-active ceph-osd@*"
+    Return From Keyword If   '${stdout}' == 'active'
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl -a reset-failed ceph-osd@*"
+    Execute Command          ${SSH_CMD} ${node} "sudo systemctl -a start ceph-osd@*"
+    ${stdout}=               Execute Command  ${SSH_CMD} ${node} "systemctl -a is-active ceph-osd@*"
+    Should Be Equal As Strings  ${stdout}  active

diff --git a/ceph/ceph_service.robot b/ceph/ceph_service.robot
index 0941b17..d28176b 100644 (file)
--- a/ceph/ceph_service.robot
+++ b/ceph/ceph_service.robot
@@ -17,6 +17,8 @@
 
 
 *** Settings ***
 
 
 *** Settings ***

+Documentation     Tests to verify the availability and recovery of failed
+...               Ceph services

 Resource          ceph_service.resource
 Suite Setup       Open Connection And Log In
 Suite Teardown    Close All Connections
 Resource          ceph_service.resource
 Suite Setup       Open Connection And Log In
 Suite Teardown    Close All Connections


@@ -43,3 +45,27 @@ Failure Of Two Monitors And Managers
     Start Ceph Manager  @{nodes}
     Sleep  10s
     Ceph Should Be Healthy
     Start Ceph Manager  @{nodes}
     Sleep  10s
     Ceph Should Be Healthy

+
+Failure Of Single Object Storage Daemon
+    @{nodes}  Create list  ${NODENAME}-1
+    ${num_up_osds}   Number Of OSDs Up
+    Kill Ceph OSD  @{nodes}
+    Sleep  5s
+    Number Of OSDs Up Should Be  ${num_up_osds-1}
+    Ceph Health Should Be Degraded
+    Start Ceph OSD  @{nodes}
+    Sleep  10s
+    Number Of OSDs Up Should Be  ${num_up_osds}
+    Ceph Should Be Healthy
+
+Failure Of Two Object Storage Daemons
+    @{nodes}  Create list  ${NODENAME}-1  ${NODENAME}-2
+    ${num_up_osds}   Number Of OSDs Up
+    Kill Ceph OSD  @{nodes}
+    Sleep  5s
+    Number Of OSDs Up Should Be  ${num_up_osds-2}
+    Ceph Health Should Be Degraded
+    Start Ceph OSD  @{nodes}
+    Sleep  10s
+    Number Of OSDs Up Should Be  ${num_up_osds}
+    Ceph Should Be Healthy