X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=blobdiff_plain;f=cmd%2Fbpa-operator%2Fe2etest%2Fbpa_vm_verifier.sh;h=cdb4e5c476902cc422ab57d49e7e1c15b88c4168;hb=e3be88235a3456ea1b7fd34460d21d894ba21871;hp=271935eb51959ae04f98c70ce52b9d2c858826a3;hpb=beed8dfcb4b78c1c05537264eada4c2212880cea;p=icn.git diff --git a/cmd/bpa-operator/e2etest/bpa_vm_verifier.sh b/cmd/bpa-operator/e2etest/bpa_vm_verifier.sh index 271935e..cdb4e5c 100755 --- a/cmd/bpa-operator/e2etest/bpa_vm_verifier.sh +++ b/cmd/bpa-operator/e2etest/bpa_vm_verifier.sh @@ -2,56 +2,10 @@ set -eu -o pipefail CLUSTER_NAME=cluster-test +ADDONS_NAMESPACE=kud NUM_MASTERS=${NUM_MASTERS:-"1"} NUM_WORKERS=${NUM_WORKERS:-"1"} -# Create Fake DHCP File -mkdir -p /opt/icn/dhcp -cat < /opt/icn/dhcp/dhcpd.leases -# The format of this file is documented in the dhcpd.leases(5) manual page. -# This lease file was written by isc-dhcp-4.3.5 - -# authoring-byte-order entry is generated, DO NOT DELETE -authoring-byte-order little-endian; - -EOF -for ((master=0;master> /opt/icn/dhcp/dhcpd.leases -lease ${ip} { - starts 4 2019/08/08 22:32:49; - ends 4 2019/08/08 23:52:49; - cltt 4 2019/08/08 22:32:49; - binding state active; - next binding state free; - rewind binding state free; - hardware ethernet ${mac}; - client-hostname "master-${master}"; -} -EOF -done -for ((worker=0;worker> /opt/icn/dhcp/dhcpd.leases -lease ${ip} { - starts 4 2019/08/08 22:32:49; - ends 4 2019/08/08 23:52:49; - cltt 4 2019/08/08 22:32:49; - binding state active; - next binding state free; - rewind binding state free; - hardware ethernet ${mac}; - client-hostname "worker-${worker}"; -} -EOF -done - # Create provisioning CR file for testing cat < e2etest/e2e_test_provisioning_cr.yaml apiVersion: bpa.akraino.org/v1alpha1 @@ -65,8 +19,7 @@ spec: masters: EOF for ((master=0;master> e2etest/e2e_test_provisioning_cr.yaml - master-${master}: mac-address: ${mac} @@ -76,8 +29,7 @@ cat <> e2etest/e2e_test_provisioning_cr.yaml workers: EOF for ((worker=0;worker> e2etest/e2e_test_provisioning_cr.yaml - worker-${worker}: mac-address: ${mac} @@ -129,45 +81,110 @@ else exit 1 fi -#Install addons -printf "Installing KUD addons\n" -pushd /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/addons -/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f prerequisites.yaml -v values.yaml -/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f composite-app.yaml -v values.yaml -popd +function wait_for { + local -r interval=30 + for ((try=0;try<600;try+=${interval})); do + echo "$(date +%H:%M:%S) - Waiting for $*" + sleep ${interval}s + if $*; then return 0; fi + done + return 1 +} -#Wait for addons to be ready -# The deployment intent group status reports instantiated before all -# Pods are ready, so wait for the instance label (.spec.version) of -# the deployment intent group instead. -status="Pending" -for try in {0..9}; do - printf "Waiting for KUD addons to be ready\n" - sleep 30s - if KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl wait pod -l app.kubernetes.io/instance=r1 --for=condition=Ready --all-namespaces --timeout=0s 2>/dev/null >/dev/null; then - status="Ready" - break - fi -done -[[ $status == "Ready" ]] +function emco_ready { + KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n emco wait pod --all --for=condition=Ready --timeout=0s 1>/dev/null 2>/dev/null +} -#Install addon resources -printf "Installing KUD addon resources\n" +function emcoctl_apply { + [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f $@ -v values.yaml | + awk '/Response Code:/ {code=$3} END{print code}') =~ 2.. ]] +} + +function emcoctl_delete { + [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f $@ -v values.yaml | + awk '/Response Code:/ {code=$3} END{print code}') =~ 404 ]] +} + +function emcoctl_instantiate { + [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/$@/v1/deployment-intent-groups/deployment/instantiate | + awk '/Response Code:/ {code=$3} END{print code}') =~ 2.. ]] +} + +function emcoctl_terminate { + [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/$@/v1/deployment-intent-groups/deployment/terminate | + awk '/Response Code:/ {code=$3} END{print code}') =~ 2.. ]] +} + +function emcoctl { + local -r op=$1 + shift + + local -r interval=2 + for ((try=0;try<600;try+=${interval})); do + if emco_ready; then break; fi + echo "$(date +%H:%M:%S) - Waiting for emco" + sleep ${interval}s + done + + for ((;try<600;try+=${interval})); do + case ${op} in + "apply") if emcoctl_apply $@; then return 0; fi ;; + "delete") if emcoctl_delete $@; then return 0; fi ;; + "instantiate") if emcoctl_instantiate $@; then return 0; fi ;; + "terminate") if emcoctl_terminate $@; then return 0; fi ;; + esac + echo "$(date +%H:%M:%S) - Waiting for emcoctl ${op} $@" + sleep ${interval}s + done + + return 1 +} + +function addons_instantiated { + KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} wait pod -l app.kubernetes.io/instance=r1 --for=condition=Ready --timeout=0s 1>/dev/null 2>/dev/null +} + +function addons_terminated { + [[ $(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get pod -l app.kubernetes.io/instance=r1 --no-headers 2>/dev/null | wc -l) == 0 ]] +} + +function networks_instantiated { + local -r count=$(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get sriovnetworknodestate --no-headers 2>/dev/null | wc -l) + local -r succeeded=$(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get sriovnetworknodestate -o jsonpath='{range .items[*]}{.status.syncStatus}{"\n"}{end}' 2>/dev/null | grep "Succeeded" | wc -l) + [[ $count == $succeeded ]] +} + +function networks_terminated { + # The syncStatus will be the same whether we are instantiating or terminating an SR-IOV network + networks_instantiated +} + +function kubevirt_instantiated { + [[ $(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get kubevirt -o jsonpath='{range .items[*]}{.status.phase}{"\n"}{end}' 2>/dev/null | grep "Deployed" | wc -l) == 1 ]] + [[ $(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get cdi -o jsonpath='{range .items[*]}{.status.phase}{"\n"}{end}' 2>/dev/null | grep "Deployed" | wc -l) == 1 ]] +} + +function kubevirt_terminated { + [[ $(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get kubevirt --no-headers 2>/dev/null | wc -l) == 0 ]] + [[ $(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl -n ${ADDONS_NAMESPACE} get cdi --no-headers 2>/dev/null | wc -l) == 0 ]] +} + +#Apply addons +printf "Applying KUD addons\n" pushd /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/addons -/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f composite-app.yaml -v values-resources.yaml +emcoctl apply 00-controllers.yaml +emcoctl apply 01-cluster.yaml +emcoctl apply 02-project.yaml +emcoctl apply 03-addons-app.yaml popd -#Wait for addon resources to be ready -status="Pending" -for try in {0..9}; do - printf "Waiting for KUD addon resources to be ready\n" - sleep 30s - if KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl wait pod -l app.kubernetes.io/instance=r1 --for=condition=Ready --all-namespaces --timeout=0s 2>/dev/null >/dev/null; then - status="Ready" - break - fi -done -[[ $status == "Ready" ]] +#Instantiate addons +emcoctl instantiate addons +wait_for addons_instantiated +emcoctl instantiate networks +wait_for networks_instantiated +emcoctl instantiate kubevirt +wait_for kubevirt_instantiated #Test addons printf "Testing KUD addons\n" @@ -176,13 +193,14 @@ failed_kud_tests="" container_runtime=$(KUBECONFIG=${CLUSTER_KUBECONFIG} kubectl get nodes -o jsonpath='{.items[].status.nodeInfo.containerRuntimeVersion}') if [[ "${container_runtime}" == "containerd://1.2.13" ]]; then #With containerd 1.2.13, the qat test container image fails to unpack. - kud_tests="topology-manager-sriov multus ovn4nfv nfd sriov-network cmk" + kud_tests="topology-manager-sriov kubevirt multus ovn4nfv nfd sriov-network cmk" else - kud_tests="topology-manager-sriov multus ovn4nfv nfd sriov-network qat cmk" + kud_tests="topology-manager-sriov kubevirt multus ovn4nfv nfd sriov-network qat cmk" fi for test in ${kud_tests}; do KUBECONFIG=${CLUSTER_KUBECONFIG} bash ${test}.sh || failed_kud_tests="${failed_kud_tests} ${test}" done +KUBECONFIG=${CLUSTER_KUBECONFIG} DEMO_FOLDER=${PWD} PATH=/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts:${PATH} bash plugin_fw_v2.sh --external || failed_kud_tests="${failed_kud_tests} plugin_fw_v2" if [[ ! -z "$failed_kud_tests" ]]; then printf "Test cases failed:${failed_kud_tests}\n" exit 1 @@ -191,25 +209,18 @@ popd printf "All test cases passed\n" #Tear down setup -printf "\n\nBeginning E2E Test Teardown\n\n" -# Workaround known issue with emcoctl resource deletion by retrying -# until a 404 is received. +printf "\n\nBeginning BMH E2E Test Teardown\n\n" +emcoctl terminate kubevirt +wait_for kubevirt_terminated +emcoctl terminate networks +wait_for networks_terminated +emcoctl terminate addons +wait_for addons_terminated pushd /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/addons -until [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f composite-app.yaml -v values-resources.yaml | - awk '/Response Code:/ {code=$3} END{print code}') =~ 404 ]]; do - echo "Waiting for KUD addon resources to terminate" - sleep 1s -done -until [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f composite-app.yaml -v values.yaml | - awk '/Response Code:/ {code=$3} END{print code}') =~ 404 ]]; do - echo "Waiting for KUD addons to terminate" - sleep 1s -done -until [[ $(/opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f prerequisites.yaml -v values.yaml | - awk '/Response Code:/ {code=$3} END{print code}') =~ 404 ]]; do - echo "Waiting for KUD addons to terminate" - sleep 1s -done +emcoctl delete 03-addons-app.yaml +emcoctl delete 02-project.yaml +emcoctl delete 01-cluster.yaml +emcoctl delete 00-controllers.yaml popd kubectl delete -f e2etest/e2e_test_provisioning_cr.yaml kubectl delete job kud-${CLUSTER_NAME} @@ -217,5 +228,4 @@ kubectl delete --ignore-not-found=true configmap ${CLUSTER_NAME}-configmap rm e2etest/e2e_test_provisioning_cr.yaml rm -rf /opt/kud/multi-cluster/${CLUSTER_NAME} rm -rf /opt/kud/multi-cluster/addons -rm /opt/icn/dhcp/dhcpd.leases make delete