From: Juha Kosonen Date: Thu, 16 May 2019 08:52:01 +0000 (+0300) Subject: Add tests for Ceph OSD running as systemd service X-Git-Tag: 2.0.0~89 X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=c71e5316007ab7763eb493cb1c5dfc246ab0ce39;p=validation.git Add tests for Ceph OSD running as systemd service - Ceph services are available when Ceph OSD is failing - Ceph OSD recovers back to operational JIRA: VAL-22 Change-Id: Id7b7324df5437cfeecc103dacb165b281e26cb0e Signed-off-by: Juha Kosonen --- diff --git a/ceph/ceph_service.resource b/ceph/ceph_service.resource index 048dbcb..62603f2 100644 --- a/ceph/ceph_service.resource +++ b/ceph/ceph_service.resource @@ -37,6 +37,11 @@ Start Ceph Monitor :FOR ${node} IN @{nodes} \ ${result}= Run Monitor Start ${node} +Start Ceph OSD + [Arguments] @{nodes} + :FOR ${node} IN @{nodes} + \ ${result}= Run Ceph OSD Start ${node} + Kill Ceph Manager [Arguments] @{nodes} :FOR ${node} IN @{nodes} @@ -47,6 +52,11 @@ Kill Ceph Monitor :FOR ${node} IN @{nodes} \ ${stdout}= Execute Command ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-mon" +Kill Ceph OSD + [Arguments] @{nodes} + :FOR ${node} IN @{nodes} + \ ${stdout}= Execute Command ${SSH_CMD} ${node} "sudo pkill -u ceph ceph-osd" + Ceph Should Be Healthy ${stdout}= Execute Command ceph health --connect-timeout 20 sudo=True Should Contain ${stdout} HEALTH_OK @@ -55,6 +65,16 @@ Ceph Health Should Be Degraded ${stdout}= Execute Command ceph health --connect-timeout 20 sudo=True Should Contain ${stdout} HEALTH_WARN +Number Of OSDs Up + ${stdout}= Execute Command ceph -f json osd stat sudo=True + ${stat}= Evaluate json.loads($stdout) json + [Return] ${stat["num_up_osds"]} + +Number Of OSDs Up Should Be + [Arguments] ${num_expected} + ${num_up_osds}= Number Of OSDs Up + Should Be Equal ${num_up_osds} ${num_expected} + Run Manager Start [Arguments] ${node} Execute Command ${SSH_CMD} ${node} "sudo systemctl start ceph-mgr@${node}" @@ -74,3 +94,13 @@ Run Monitor Start Execute Command ${SSH_CMD} ${node} "sudo systemctl start ceph-mon@${node}" ${stdout}= Execute Command ${SSH_CMD} ${node} "systemctl is-active ceph-mon@${node}" Should Be Equal As Strings ${stdout} active + +Run Ceph OSD Start + [Arguments] ${node} + Execute Command ${SSH_CMD} ${node} "sudo systemctl -a start ceph-osd@*" + ${stdout}= Execute Command ${SSH_CMD} ${node} "systemctl -a is-active ceph-osd@*" + Return From Keyword If '${stdout}' == 'active' + Execute Command ${SSH_CMD} ${node} "sudo systemctl -a reset-failed ceph-osd@*" + Execute Command ${SSH_CMD} ${node} "sudo systemctl -a start ceph-osd@*" + ${stdout}= Execute Command ${SSH_CMD} ${node} "systemctl -a is-active ceph-osd@*" + Should Be Equal As Strings ${stdout} active diff --git a/ceph/ceph_service.robot b/ceph/ceph_service.robot index 0941b17..d28176b 100644 --- a/ceph/ceph_service.robot +++ b/ceph/ceph_service.robot @@ -17,6 +17,8 @@ *** Settings *** +Documentation Tests to verify the availability and recovery of failed +... Ceph services Resource ceph_service.resource Suite Setup Open Connection And Log In Suite Teardown Close All Connections @@ -43,3 +45,27 @@ Failure Of Two Monitors And Managers Start Ceph Manager @{nodes} Sleep 10s Ceph Should Be Healthy + +Failure Of Single Object Storage Daemon + @{nodes} Create list ${NODENAME}-1 + ${num_up_osds} Number Of OSDs Up + Kill Ceph OSD @{nodes} + Sleep 5s + Number Of OSDs Up Should Be ${num_up_osds-1} + Ceph Health Should Be Degraded + Start Ceph OSD @{nodes} + Sleep 10s + Number Of OSDs Up Should Be ${num_up_osds} + Ceph Should Be Healthy + +Failure Of Two Object Storage Daemons + @{nodes} Create list ${NODENAME}-1 ${NODENAME}-2 + ${num_up_osds} Number Of OSDs Up + Kill Ceph OSD @{nodes} + Sleep 5s + Number Of OSDs Up Should Be ${num_up_osds-2} + Ceph Health Should Be Degraded + Start Ceph OSD @{nodes} + Sleep 10s + Number Of OSDs Up Should Be ${num_up_osds} + Ceph Should Be Healthy