From 6a857e160806837546157bd2d44bc9d5565dd3d3 Mon Sep 17 00:00:00 2001 From: Itohan Date: Tue, 15 Oct 2019 13:46:42 -0700 Subject: [PATCH] Added e2e scripts and modified Makefile - Fixed Bug in Makefile and operator.yaml - Added bpa delete option to Makefile - Added e2e test and bpa_op verifier - Added e2e testing files Signed-off-by: Itohan Ukponmwan Change-Id: I006d8e4967eaee9b47b24b05b3aeadaef5432476 --- Makefile | 19 +- cmd/bpa-operator/Makefile | 16 +- cmd/bpa-operator/deploy/operator.yaml | 6 +- cmd/bpa-operator/e2etest/Vagrantfile | 81 ++++++ cmd/bpa-operator/e2etest/bmh-bpa-test.yaml | 290 +++++++++++++++++++++ cmd/bpa-operator/e2etest/bpa_e2e_test.sh | 33 +++ .../e2etest/e2e_test_provisioning_cr.yaml | 11 + cmd/bpa-operator/e2etest/fake_dhcp_lease | 36 +++ cmd/bpa-operator/e2etest/vm_authorized_keys | 1 + 9 files changed, 485 insertions(+), 8 deletions(-) create mode 100644 cmd/bpa-operator/e2etest/Vagrantfile create mode 100644 cmd/bpa-operator/e2etest/bmh-bpa-test.yaml create mode 100755 cmd/bpa-operator/e2etest/bpa_e2e_test.sh create mode 100644 cmd/bpa-operator/e2etest/e2e_test_provisioning_cr.yaml create mode 100644 cmd/bpa-operator/e2etest/fake_dhcp_lease create mode 100644 cmd/bpa-operator/e2etest/vm_authorized_keys diff --git a/Makefile b/Makefile index 6b2ff21..09333fc 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ BMDIR:=$(CURDIR)/env/metal3 METAL3DIR:=$(CURDIR)/deploy/metal3/scripts BPA_OPERATOR:=$(CURDIR)/cmd/bpa-operator/ KUD_PATH:=$(CURDIR)/deploy/kud +BPA_E2E_SETUP:=https://raw.githubusercontent.com/onap/multicloud-k8s/master/kud/hosting_providers/vagrant/setup.sh all: bm_install @@ -10,16 +11,28 @@ bm_preinstall: pushd $(BMDIR) && ./01_install_package.sh && ./02_configure.sh && ./03_launch_prereq.sh && popd bm_install: - pushd $(METAL3DIR) && ./metal3.sh && popd + pushd $(METAL3DIR) && ./metal3.sh && popd bm_all: bm_preinstall bm_install kud_download: pushd $(KUD_PATH) && ./kud_launch.sh && popd -bpa_op_install: kud_download +bpa_op_install: pushd $(BPA_OPERATOR) && make docker && make deploy && popd -bpa_op_all: bm_all bpa_op_install +bpa_op_delete: + pushd $(BPA_OPERATOR) && make delete && popd + +bpa_op_e2e_preinstall: + wget $(BPA_E2E_SETUP) && bash setup.sh -p libvirt + +bpa_op_e2e: + pushd $(BPA_OPERATOR) && make e2etest && popd + +bpa_op_verifier: bpa_op_install bpa_op_e2e + +bpa_op_all: bm_all bpa_op_install + .PHONY: all bm_preinstall bm_install diff --git a/cmd/bpa-operator/Makefile b/cmd/bpa-operator/Makefile index 51ca2af..a023e13 100644 --- a/cmd/bpa-operator/Makefile +++ b/cmd/bpa-operator/Makefile @@ -10,5 +10,19 @@ deploy: kubectl apply -f deploy/service_account.yaml kubectl apply -f deploy/role.yaml kubectl apply -f deploy/role_binding.yaml - kubectl apply -f deploy/crds/bpa_v1alpha1_provisioning_crd.yaml + kubectl apply -f deploy/crds/provisioning-crd/bpa_v1alpha1_provisioning_crd.yaml + kubectl apply -f deploy/crds/software-crd/bpa_v1alpha1_software_crd.yaml kubectl apply -f deploy/operator.yaml + +.PHONY: delete +delete: + kubectl delete -f deploy/service_account.yaml + kubectl delete -f deploy/role.yaml + kubectl delete -f deploy/role_binding.yaml + kubectl delete -f deploy/crds/provisioning-crd/bpa_v1alpha1_provisioning_crd.yaml + kubectl delete -f deploy/crds/software-crd/bpa_v1alpha1_software_crd.yaml + kubectl delete -f deploy/operator.yaml + +.PHONY: e2etest +e2etest: + cd e2etest && ./bpa_e2e_test.sh diff --git a/cmd/bpa-operator/deploy/operator.yaml b/cmd/bpa-operator/deploy/operator.yaml index d7acdc2..0ce498c 100644 --- a/cmd/bpa-operator/deploy/operator.yaml +++ b/cmd/bpa-operator/deploy/operator.yaml @@ -22,8 +22,6 @@ spec: volumeMounts: - name: dhcp-shared mountPath: /var/lib/dhcp/ - - name: icn-kud - mountPath: /multicloud-k8s - name: icn-cluster mountPath: /multi-cluster command: @@ -45,6 +43,6 @@ spec: - name: dhcp-shared hostPath: path: /opt/icn/dhcp/ - - name: icn-kud + - name: icn-cluster hostPath: - path: /opt/icn/multicloud-k8s/ + path: /opt/kud/multi-cluster diff --git a/cmd/bpa-operator/e2etest/Vagrantfile b/cmd/bpa-operator/e2etest/Vagrantfile new file mode 100644 index 0000000..8ee0d71 --- /dev/null +++ b/cmd/bpa-operator/e2etest/Vagrantfile @@ -0,0 +1,81 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => "elastic/ubuntu-18.04-x86_64", :version=> '20191020.0.0' }, + :libvirt => { :name => "generic/ubuntu1804", :version=> "1.9.38"} +} + + + +provider = (ENV["VAGRANT_DEFAULT_PROVIDER"] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV["no_proxy"] != nil or ENV["NO_PROXY"] + $no_proxy = ENV["NO_PROXY"] || ENV["no_proxy"] || "127.0.0.1,localhost,192.168.50.63" + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.ssh.insert_key = false + config.vm.box_version = box[provider][:version] + config.vm.provision "file", source: "vm_authorized_keys", destination: "/home/vagrant/.ssh/authorized_keys" + + config.vm.provision "shell" do |sh| + sh.inline = "mkdir -p /root/.ssh; cp /home/vagrant/.ssh/authorized_keys /root/.ssh/authorized_keys" + sh.privileged = true + end + + if ENV["http_proxy"] != nil and ENV["https_proxy"] != nil + if Vagrant.has_plugin?("vagrant-proxyconf") + config.proxy.http = ENV["http_proxy"] || ENV["HTTP_PROXY"] || "" + config.proxy.https = ENV["https_proxy"] || ENV["HTTPS_PROXY"] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider "libvirt" do |v| + v.nested = true + v.cpu_mode = "host-passthrough" + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + + config.vm.define vm_name = "bpa-test-vm" do |config| + config.vm.hostname = "master-test" + config.vm.network :private_network, :ip => "192.168.50.63", :type => :static, :mac => "080027efab60" + end + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", 16384] + v.customize ["modifyvm", :id, "--cpus", 16] + end + + config.vm.provider "libvirt" do |v| + v.memory = 16384 + v.cpus = 16 + + end + #end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end +end diff --git a/cmd/bpa-operator/e2etest/bmh-bpa-test.yaml b/cmd/bpa-operator/e2etest/bmh-bpa-test.yaml new file mode 100644 index 0000000..1ce7b47 --- /dev/null +++ b/cmd/bpa-operator/e2etest/bmh-bpa-test.yaml @@ -0,0 +1,290 @@ +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: bpa-test-bmh +spec: + online: true + bmc: + address: ipmi://172.31.1.17 + credentialsName: demo-bmc-secret + image: + url: "http://172.22.0.1/images/bionic-server-cloudimg-amd64.img" + checksum: "http://172.22.0.1/images/bionic-server-cloudimg-amd64.md5sum" + userData: + name: demo-user-data + namespace: metal3 + spec: + bmc: + address: ipmi://172.31.1.17 + credentialsName: demo-bmc-secret + bootMACAddress: "3c:fd:fe:9c:88:63" + description: "" + externallyProvisioned: false + hardwareProfile: "" + image: + checksum: http://172.22.0.1/images/bionic-server-cloudimg-amd64.md5sum + url: http://172.22.0.1/images/bionic-server-cloudimg-amd64.img + online: true + userData: + name: demo-user-data + namespace: metal3 + status: + errorMessage: "" + goodCredentials: + credentials: + name: demo-bmc-secret + namespace: metal3 + credentialsVersion: "584398" + hardware: + cpu: + arch: x86_64 + clockMegahertz: 3700 + count: 72 + flags: + - 3dnowprefetch + - abm + - acpi + - adx + - aes + - aperfmperf + - apic + - arat + - arch_perfmon + - art + - avx + - avx2 + - avx512bw + - avx512cd + - avx512dq + - avx512f + - avx512vl + - bmi1 + - bmi2 + - bts + - cat_l3 + - cdp_l3 + - clflush + - clflushopt + - clwb + - cmov + - constant_tsc + - cqm + - cqm_llc + - cqm_mbm_local + - cqm_mbm_total + - cqm_occup_llc + - cx16 + - cx8 + - dca + - de + - ds_cpl + - dtes64 + - dtherm + - dts + - eagerfpu + - epb + - ept + - erms + - est + - f16c + - flexpriority + - flush_l1d + - fma + - fpu + - fsgsbase + - fxsr + - hle + - ht + - hwp + - hwp_act_window + - hwp_epp + - hwp_pkg_req + - ibpb + - ibrs + - ida + - intel_ppin + - intel_pt + - intel_stibp + - invpcid + - lahf_lm + - lm + - mba + - mca + - mce + - md_clear + - mmx + - monitor + - movbe + - mpx + - msr + - mtrr + - nonstop_tsc + - nopl + - nx + - ospke + - pae + - pat + - pbe + - pcid + - pclmulqdq + - pdcm + - pdpe1gb + - pebs + - pge + - pku + - pln + - pni + - popcnt + - pse + - pse36 + - pts + - rdrand + - rdseed + - rdt_a + - rdtscp + - rep_good + - rtm + - sdbg + - sep + - smap + - smep + - smx + - spec_ctrl + - ss + - ssbd + - sse + - sse2 + - sse4_1 + - sse4_2 + - ssse3 + - stibp + - syscall + - tm + - tm2 + - tpr_shadow + - tsc + - tsc_adjust + - tsc_deadline_timer + - vme + - vmx + - vnmi + - vpid + - x2apic + - xgetbv1 + - xsave + - xsavec + - xsaveopt + - xtopology + - xtpr + model: Intel(R) Xeon(R) Gold 6140M CPU @ 2.30GHz + firmware: + bios: + date: 11/07/2018 + vendor: Intel Corporation + version: SE5C620.86B.00.01.0015.110720180833 + hostname: localhost.localdomain + nics: + - ip: "" + mac: 08:00:27:00:ab:c0 + model: 0x8086 0x1572 + name: eth3 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: a4:bf:01:64:86:6e + model: 0x8086 0x37d2 + name: eth4 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:88:62 + model: 0x8086 0x1572 + name: eth2 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:88:60 + model: 0x8086 0x1572 + name: eth0 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:88:61 + model: 0x8086 0x1572 + name: eth1 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:87:63 + model: 0x8086 0x1572 + name: eth9 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:87:62 + model: 0x8086 0x1572 + name: eth8 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:87:60 + model: 0x8086 0x1572 + name: eth6 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: "" + mac: 3c:fd:fe:9c:87:61 + model: 0x8086 0x1572 + name: eth7 + pxe: false + speedGbps: 0 + vlanId: 0 + - ip: 172.22.0.54 + mac: a4:bf:01:64:86:6f + model: 0x8086 0x37d2 + name: eth5 + pxe: true + speedGbps: 0 + vlanId: 0 + ramMebibytes: 262144 + storage: + - hctl: "6:0:0:0" + model: INTEL SSDSC2KB48 + name: /dev/sda + rotational: false + serialNumber: BTYF8290022M480BGN + sizeBytes: 480103981056 + vendor: ATA + wwn: "0x55cd2e414fc888c1" + wwnWithExtension: "0x55cd2e414fc888c1" + - hctl: "7:0:0:0" + model: INTEL SSDSC2KB48 + name: /dev/sdb + rotational: false + serialNumber: BTYF83160FDB480BGN + sizeBytes: 480103981056 + vendor: ATA + wwn: "0x55cd2e414fd7b5a3" + wwnWithExtension: "0x55cd2e414fd7b5a3" + systemVendor: + manufacturer: Intel Corporation + productName: S2600WFT (SKU Number) + serialNumber: BQPW84200264 + hardwareProfile: unknown + lastUpdated: "2019-07-25T01:52:55Z" + operationalStatus: OK + poweredOn: true + provisioning: + ID: 36b462c1-02a0-499e-b4f1-cc6087a1c574 + image: + checksum: http://172.22.0.1/images/bionic-server-cloudimg-amd64.md5sum + url: http://172.22.0.1/images/bionic-server-cloudimg-amd64.img + state: provisioned diff --git a/cmd/bpa-operator/e2etest/bpa_e2e_test.sh b/cmd/bpa-operator/e2etest/bpa_e2e_test.sh new file mode 100755 index 0000000..13fb588 --- /dev/null +++ b/cmd/bpa-operator/e2etest/bpa_e2e_test.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cp fake_dhcp_lease /opt/icn/dhcp/dhcpd.leases +kubectl apply -f bmh-bpa-test.yaml +cat /root/.ssh/id_rsa.pub > vm_authorized_keys +vagrant up +sleep 5 +kubectl apply -f e2e_test_provisioning_cr.yaml +sleep 2 +status="Running" + +while [[ $status == "Running" ]] +do + stats=$(kubectl get pods |grep -i kud-cluster-test) + + status=$(echo $stats | cut -d " " -f 3) + echo "KUD install job still running" + sleep 2m +done + +if [[ $status == "Completed" ]]; +then + printf "KUD Install completed successfully\n" +else + printf "KUD Install failed\n" +fi + +printf "\n\nBeginning E2E Test Teardown\n\n" + +kubectl delete -f e2e_test_provisioning_cr.yaml +kubectl delete -f bmh-bpa-test.yaml +kubectl delete job kud-cluster-test +vagrant destroy -f diff --git a/cmd/bpa-operator/e2etest/e2e_test_provisioning_cr.yaml b/cmd/bpa-operator/e2etest/e2e_test_provisioning_cr.yaml new file mode 100644 index 0000000..4e2a7d5 --- /dev/null +++ b/cmd/bpa-operator/e2etest/e2e_test_provisioning_cr.yaml @@ -0,0 +1,11 @@ +apiVersion: bpa.akraino.org/v1alpha1 +kind: Provisioning +metadata: + name: e2e-test-provisioning + labels: + cluster: cluster-test + owner: c1 +spec: + masters: + - master-test: + mac-address: 08:00:27:00:ab:c0 diff --git a/cmd/bpa-operator/e2etest/fake_dhcp_lease b/cmd/bpa-operator/e2etest/fake_dhcp_lease new file mode 100644 index 0000000..22b9c49 --- /dev/null +++ b/cmd/bpa-operator/e2etest/fake_dhcp_lease @@ -0,0 +1,36 @@ +# The format of this file is documented in the dhcpd.leases(5) manual page. +# This lease file was written by isc-dhcp-4.3.5 + +# authoring-byte-order entry is generated, DO NOT DELETE +authoring-byte-order little-endian; + +lease 192.168.50.63 { + starts 4 2019/08/08 22:32:49; + ends 4 2019/08/08 23:52:49; + cltt 4 2019/08/08 22:32:49; + binding state active; + next binding state free; + rewind binding state free; + hardware ethernet 08:00:27:00:ab:c0; + client-hostname "master-test"; +} +lease 192.168.50.64 { + starts 4 2019/08/08 22:32:49; + ends 4 2019/08/08 23:52:49; + cltt 4 2019/08/08 22:32:49; + binding state active; + next binding state free; + rewind binding state free; + hardware ethernet 00:c4:13:04:62:b5; + client-hostname "bpa-test-2"; +} +lease 192.168.50.65 { + starts 4 2019/08/08 22:32:49; + ends 4 2019/08/08 23:52:49; + cltt 4 2019/08/08 22:32:49; + binding state active; + next binding state free; + rewind binding state free; + hardware ethernet 00:c3:11:02:61:b5; + client-hostname "bpa-test-3"; +} diff --git a/cmd/bpa-operator/e2etest/vm_authorized_keys b/cmd/bpa-operator/e2etest/vm_authorized_keys new file mode 100644 index 0000000..432f9e1 --- /dev/null +++ b/cmd/bpa-operator/e2etest/vm_authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6e+WXE8sTlmW1jukGFbHKuqOgTY/ZvVh94LClzxGlVt9rPc8sd3Pqjoy1NB40WCqwdixmBEfsj0VPTPnZjHp9Qkh5MS4CBLkmJFtIcUBCDGSJ+PF8rzX5W8/TRvmWAmeGJOkyxiq+GxSEn2OCMoCpX4mz32nbInQcR5yIEPw0EYdtd3QmpLHiR6mf57rGTnr05jOJdUhLuF2I9gviy7FKVm7VnWxGzAbp/XiG3xT2NBpZ+TY4wg4tYa768wP8CKD8ixpfJNPsMaWBSgQOWG8hrYgC206utT5t0cwUJMExUFBSUwck8XpH1DP2/ce5rUtkdHnr6ve8RpyR7fOtB9YH root@AP11-29 -- 2.16.6