From c3b53a32784bfcb07e6d5dbe84b1b365ff691a4a Mon Sep 17 00:00:00 2001 From: "r.kuralamudhan" Date: Fri, 8 Nov 2019 00:18:56 +0000 Subject: [PATCH] adding bm verifier Signed-off-by: r.kuralamudhan Change-Id: I61da26d438ca57f1a08b0cf0e86699b58367dbbc --- Makefile | 26 +++++--- deploy/metal3/scripts/01_metal3.sh | 21 +++++++ deploy/metal3/scripts/03_verify_deprovisioning.sh | 76 +++++++++++++++++++++++ deploy/metal3/scripts/nodes.json.sample | 12 ++-- env/lib/common.sh | 19 ++++-- env/metal3/06_host_cleanup.sh | 5 ++ user_config.sh | 12 ++++ 7 files changed, 153 insertions(+), 18 deletions(-) create mode 100755 deploy/metal3/scripts/03_verify_deprovisioning.sh create mode 100644 user_config.sh diff --git a/Makefile b/Makefile index fcb28f7..3af3ff9 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ KUD_PATH:=$(CURDIR)/deploy/kud SDWAN_VERIFIER_PATH:=$(CURDIR)/sdwan/test BPA_REST_API:=$(CURDIR)/cmd/bpa-restapi-agent - help: @echo " Targets:" @echo " test -- run unit tests" @@ -17,15 +16,26 @@ help: @echo " unit -- run the unit tests" @echo " help -- this help output" -all: bm_install +install: bmh_all + +bmh_preinstall: + pushd $(BMDIR) && ./01_install_package.sh && ./02_configure.sh && \ + ./03_launch_prereq.sh && popd + +bmh_clean: + pushd $(METAL3DIR) && ./01_metal3.sh deprovision && \ + ./03_verify_deprovisioning.sh && ./01_metal3.sh clean popd -bm_preinstall: - pushd $(BMDIR) && ./01_install_package.sh && ./02_configure.sh && ./03_launch_prereq.sh && popd +bmh_clean_host: + pushd $(BMDIR) && ./06_host_cleanup.sh && popd -bm_install: - pushd $(METAL3DIR) && ./metal3.sh && popd +bmh_install: + pushd $(METAL3DIR) && ./01_metal3.sh launch && \ + ./01_metal3.sh provision && ./02_verify.sh && popd -bm_all: bm_preinstall bm_install +bmh_all: bmh_preinstall bmh_install + +bmh_clean_all: bmh_clean bmh_clean_host kud_bm_deploy_mini: pushd $(KUD_PATH) && ./kud_bm_launch.sh minimal && popd @@ -76,6 +86,8 @@ bashate: prerequisite: pushd $(ENV) && ./cd_package_installer.sh && popd +bm_verifer: install + verify_all: prerequisite \ metal3_prerequisite \ kud_bm_deploy_mini \ diff --git a/deploy/metal3/scripts/01_metal3.sh b/deploy/metal3/scripts/01_metal3.sh index fa9e2ec..0cc7ba0 100755 --- a/deploy/metal3/scripts/01_metal3.sh +++ b/deploy/metal3/scripts/01_metal3.sh @@ -174,6 +174,18 @@ function remove_bm_hosts { done } +function cleanup { + while read -r name username password address; do + kubectl delete bmh $name -n metal3 + kubectl delete secrets $name-bmc-secret -n metal3 + kubectl delete secrets $name-user-data -n metal3 + done +} + +function clean_all { + list_nodes | cleanup +} + function apply_bm_hosts { list_nodes | make_bm_hosts } @@ -188,19 +200,28 @@ if [ "$1" == "launch" ]; then fi if [ "$1" == "deprovision" ]; then + configure_nodes deprovision_all_hosts exit 0 fi if [ "$1" == "provision" ]; then + configure_nodes apply_bm_hosts exit 0 fi +if [ "$1" == "clean" ]; then + configure_nodes + clean_all + exit 0 +fi + echo "Usage: metal3.sh" echo "launch - Launch the metal3 operator" echo "provision - provision baremetal node as specified in common.sh" echo "deprovision - deprovision baremetal node as specified in common.sh" +echo "clean - clean all the resources" exit 1 #Following code is tested for the offline mode diff --git a/deploy/metal3/scripts/03_verify_deprovisioning.sh b/deploy/metal3/scripts/03_verify_deprovisioning.sh new file mode 100755 index 0000000..97a8a21 --- /dev/null +++ b/deploy/metal3/scripts/03_verify_deprovisioning.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +#set -x + +LIBDIR="$(dirname "$(dirname "$(dirname "$PWD")")")" + +eval "$(go env)" + +source $LIBDIR/env/lib/common.sh + +declare -i timeout=60 +declare -i interval=60 + +function check_deprovisioned { + declare -i prev_host_state=0 + declare -i j=0 + echo "Baremetal state: 1 means deprovisioned & 0 means not yet deprovisioned" + while read -r name username password address; do + declare -i current_host_state=0 + state=$(kubectl get baremetalhosts $name -n metal3 -o json | jq -r '.status.provisioning.state') + echo "Baremetal host metal3 state - "$name" : "$state + + if [ $state == "ready" ];then + current_host_state=1 + fi + + echo "Baremetal $name current_host_state : "$current_host_state + echo "Previous Baremetals prev_host_state : "$prev_host_state + + if [ $j -eq 0 ]; then + prev_host_state=$current_host_state + ((j+=1)) + continue + fi + + if [ $current_host_state -eq 1 ] && [ $prev_host_state -eq 1 ]; then + prev_host_state=1 + else + prev_host_state=0 + fi + + echo "All Baremetal hosts aggregated state - prev_host_state:"$prev_host_state + ((j+=1)) + done + return $prev_host_state +} + +function warm_up_time { + echo "Wait for 240s for all baremetal hosts to reboot and network is up" + sleep 4m +} + +function wait_for_deprovisioned { + all_bmh_deprovisioned=1 + declare -i k=1 + while ((timeout > 0)); do + echo "Try $k iteration : Wait for $interval seconds to check all bmh state" + sleep $interval + list_nodes | check_deprovisioned + all_bmh_state=$? + if [[ $all_bmh_state -eq $all_bmh_deprovisioned ]]; then + echo "All the Baremetal hosts are deprovisioned - success" + warm_up_time + exit 0 + fi + ((timeout-=1)) + ((k+=1)) + done + exit 1 +} + + +function verify_bm_hosts_cleanup { + wait_for_deprovisioned +} + +verify_bm_hosts_cleanup diff --git a/deploy/metal3/scripts/nodes.json.sample b/deploy/metal3/scripts/nodes.json.sample index d1c81c3..c9edd01 100644 --- a/deploy/metal3/scripts/nodes.json.sample +++ b/deploy/metal3/scripts/nodes.json.sample @@ -3,9 +3,9 @@ { "name": "edge01-node01", "ipmi_driver_info": { - "username": "admin", - "password": "admin", - "address": "10.10.10.11" + "username": "root", + "password": "root", + "address": "10.10.110.11" }, "os": { "image_name": "bionic-server-cloudimg-amd64.img", @@ -16,9 +16,9 @@ { "name": "edge01-node02", "ipmi_driver_info": { - "username": "admin", - "password": "admin", - "address": "10.10.10.12" + "username": "root", + "password": "root", + "address": "10.10.110.13" }, "os": { "image_name": "bionic-server-cloudimg-amd64.img", diff --git a/env/lib/common.sh b/env/lib/common.sh index 4b8471b..09b548e 100755 --- a/env/lib/common.sh +++ b/env/lib/common.sh @@ -1,5 +1,9 @@ #!/bin/bash +ICNDIR="$(dirname "$(dirname "$PWD")")" +echo $ICNDIR +source $ICNDIR/user_config.sh + #supported OS version UBUNTU_BIONIC=${UBUNTU_BIONIC:-Ubuntu 18.04.2 LTS} @@ -17,8 +21,8 @@ POD_NETWORK_CIDR=${POD_NETWORK_CIDR:-"10.244.0.0/16"} PODMAN_CNI_CONFLIST=${PODMAN_CNI_CONFLIST:-"https://raw.githubusercontent.com/containers/libpod/v1.4.4/cni/87-podman-bridge.conflist"} #Bootstrap K8s cluster -BS_DHCP_INTERFACE=${BS_DHCP_INTERFACE:-"ens513f0"} -BS_DHCP_INTERFACE_IP=${BS_DHCP_INTERFACE_IP:-"172.31.1.1/24"} +BS_DHCP_INTERFACE=${BS_DHCP_INTERFACE:-} +BS_DHCP_INTERFACE_IP=${BS_DHCP_INTERFACE_IP:-} BS_DHCP_DIR=${BS_DHCP_DIR:-$DOWNLOAD_PATH/dhcp} #Ironic variables @@ -29,11 +33,11 @@ IRONIC_BAREMETAL_SOCAT_IMAGE=${IRONIC_BAREMETAL_SOCAT_IMAGE:-"alpine/socat:lates IRONIC_DATA_DIR=${IRONIC_DATA_DIR:-"/opt/ironic"} #IRONIC_PROVISIONING_INTERFACE is required to be provisioning, don't change it -IRONIC_INTERFACE=${IRONIC_INTERFACE:-"enp4s0f1"} +IRONIC_INTERFACE=${IRONIC_INTERFACE:-} IRONIC_PROVISIONING_INTERFACE=${IRONIC_PROVISIONING_INTERFACE:-"provisioning"} -IRONIC_IPMI_INTERFACE=${IRONIC_IPMI_INTERFACE:-"enp4s0f0"} +IRONIC_IPMI_INTERFACE=${IRONIC_IPMI_INTERFACE:-} IRONIC_PROVISIONING_INTERFACE_IP=${IRONIC_PROVISIONING_INTERFACE_IP:-"172.22.0.1"} -IRONIC_IPMI_INTERFACE_IP=${IRONIC_IPMI_INTERFACE_IP:-"10.10.110.20"} +IRONIC_IPMI_INTERFACE_IP=${IRONIC_IPMI_INTERFACE_IP:-} BM_IMAGE_URL=${BM_IMAGE_URL:-"https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img"} BM_IMAGE=${BM_IMAGE:-"bionic-server-cloudimg-amd64.img"} @@ -74,6 +78,11 @@ function call_api { function list_nodes { NODES_FILE="${IRONIC_DATA_DIR}/nodes.json" + + if [ ! -f $IRONIC_DATA_DIR/nodes.json ]; then + exit 1 + fi + cat "$NODES_FILE" | \ jq '.nodes[] | { name, diff --git a/env/metal3/06_host_cleanup.sh b/env/metal3/06_host_cleanup.sh index 03fca87..9e8fbde 100755 --- a/env/metal3/06_host_cleanup.sh +++ b/env/metal3/06_host_cleanup.sh @@ -23,3 +23,8 @@ brctl delbr dhcp0 rm -rf ${BS_DHCP_DIR} rm -rf ${IRONIC_DATA_DIR} + +kubeadm reset -f +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X +rm -rf $HOME/.kube/config +rm -rf /var/lib/etcd diff --git a/user_config.sh b/user_config.sh new file mode 100644 index 0000000..2b53a51 --- /dev/null +++ b/user_config.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +#Local controller - Bootstrap cluster DHCP connection +BS_DHCP_INTERFACE=${BS_DHCP_INTERFACE:-"ens513f0"} +BS_DHCP_INTERFACE_IP=${BS_DHCP_INTERFACE_IP:-"172.31.1.1/24"} + +#Ironic Metal3 settings for provisioning network +IRONIC_INTERFACE=${IRONIC_INTERFACE:-"enp4s0f1"} + +#Ironic Metal3 setting for IPMI LAN Network +IRONIC_IPMI_INTERFACE=${IRONIC_IPMI_INTERFACE:-"enp4s0f0"} +IRONIC_IPMI_INTERFACE_IP=${IRONIC_IPMI_INTERFACE_IP:-"10.10.110.20"} -- 2.16.6