cmd/bpa-operator/e2etest/bpa_virtletvm_verifier.sh

   1 #!/usr/bin/env bash
   2 set -eu -o pipefail
   3
   4 printf "\n\nStart Virtlet VM provisioning E2E test\n\n"
   5
   6 TUNING_DIR="/tmp/tuning_dir"
   7 CNI_PLUGINS="cni-plugins-linux-amd64-v0.8.2.tgz"
   8 if !(wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/$CNI_PLUGINS -P $TUNING_DIR 2>/dev/null); then
   9     echo "Error downloading cni plugins for Virtlet VM provisioning"
  10     exit 1
  11 fi
  12
  13 pushd $TUNING_DIR
  14 if [ -f $CNI_PLUGINS ]; then
  15     tar -xzvf $CNI_PLUGINS > /dev/null
  16     if [ -f "tuning" ]; then
  17         cp "tuning" "/opt/cni/bin/"
  18         echo "Updated the tuning plugin"
  19     else
  20         echo "Error finding the latest tuning plugin"
  21         rm -rf $TUNING_DIR
  22         exit 1
  23     fi
  24     rm -rf $TUNING_DIR
  25 fi
  26 popd
  27
  28 # Create network attachment definition
  29 BPA_DIR="/tmp/bpa"
  30 mkdir -p $BPA_DIR
  31 cat <<EOF > $BPA_DIR/netattachdef-flannel-vm.yaml
  32 apiVersion: "k8s.cni.cncf.io/v1"
  33 kind: NetworkAttachmentDefinition
  34 metadata:
  35   name: flannel-vm
  36 spec:
  37   config: '{
  38             "cniVersion": "0.3.1",
  39             "name" : "cni0",
  40             "plugins": [ {
  41               "type": "flannel",
  42               "cniVersion": "0.3.1",
  43               "masterplugin": true,
  44               "delegate": {
  45                   "isDefaultGateway": true
  46               }
  47             },
  48             {
  49               "type": "tuning"
  50             }]
  51           }'
  52 EOF
  53
  54 cat <<'EOF' > $BPA_DIR/virtlet_test_vm.yaml
  55 apiVersion: apps/v1
  56 kind: Deployment
  57 metadata:
  58   name: virtlet-deployment
  59   labels:
  60     app: virtlet
  61 spec:
  62   replicas: 1
  63   selector:
  64     matchLabels:
  65       app: virtlet
  66   template:
  67     metadata:
  68       labels:
  69         app: virtlet
  70       annotations:
  71         VirtletLibvirtCPUSetting: |
  72           mode: host-passthrough
  73         # This tells CRI Proxy that this pod belongs to Virtlet runtime
  74         kubernetes.io/target-runtime: virtlet.cloud
  75         VirtletCloudInitUserData: |
  76           ssh_pwauth: True
  77           disable_root: false
  78           chpasswd: {expire: False}
  79           manage_resolv_conf: True
  80           resolv_conf:
  81             nameservers: ['8.8.8.8', '8.8.4.4']
  82           users:
  83           - name: root
  84             gecos: User
  85             primary-group: root
  86             groups: users
  87             lock_passwd: false
  88             shell: /bin/bash
  89             sudo: ALL=(ALL) NOPASSWD:ALL
  90             ssh_authorized_keys:
  91               $ssh_key
  92           runcmd:
  93             - sed -i -e 's/^#DNS=.*/DNS=8.8.8.8/g' /etc/systemd/resolved.conf
  94             - systemctl daemon-reload
  95             - systemctl restart systemd-resolved
  96         v1.multus-cni.io/default-network: '[
  97             { "name": "flannel-vm",
  98               "mac": "c2:b4:57:49:47:f1" }]'
  99         VirtletRootVolumeSize: 8Gi
 100         VirtletVCPUCount: "2"
 101     spec:
 102       affinity:
 103         nodeAffinity:
 104           requiredDuringSchedulingIgnoredDuringExecution:
 105             nodeSelectorTerms:
 106             - matchExpressions:
 107               - key: extraRuntime
 108                 operator: In
 109                 values:
 110                 - virtlet
 111       containers:
 112       - name: virtlet-deployment
 113         # This specifies the image to use.
 114         # virtlet.cloud/ prefix is used by CRI proxy, the remaining part
 115         # of the image name is prepended with https:// and used to download the image
 116         image: virtlet.cloud/ubuntu/18.04
 117         imagePullPolicy: IfNotPresent
 118         # tty and stdin required for "kubectl attach -t" to work
 119         tty: true
 120         stdin: true
 121         resources:
 122           requests:
 123             cpu: 2
 124             memory: 8Gi
 125           limits:
 126             # This memory limit is applied to the libvirt domain definition
 127             cpu: 2
 128             memory: 8Gi
 129 EOF
 130
 131 # Create provisioning CR file for BPA testing
 132 cat <<EOF > $BPA_DIR/e2e_bpa_test.yaml
 133 apiVersion: bpa.akraino.org/v1alpha1
 134 kind: Provisioning
 135 metadata:
 136   name: vmcluster110
 137   labels:
 138     cluster: vmcluster110
 139     cluster-type: virtlet-vm
 140     owner: c1
 141 spec:
 142   masters:
 143     - master-1:
 144         mac-address: c2:b4:57:49:47:f1
 145   PodSubnet: 172.21.64.0/18
 146 EOF
 147
 148 pushd $BPA_DIR
 149 # create flannel-vm net-attach-def
 150 kubectl apply -f netattachdef-flannel-vm.yaml -n kube-system
 151
 152 # generate user ssh key
 153 if [ ! -f "/root/.ssh/id_rsa.pub" ]; then
 154     ssh-keygen -f /root/.ssh/id_rsa -P ""
 155 fi
 156
 157 # create ssh key secret
 158 kubectl create secret generic ssh-key-secret --from-file=id_rsa=/root/.ssh/id_rsa --from-file=id_rsa.pub=/root/.ssh/id_rsa.pub
 159
 160 # create virtlet vm
 161 key=$(cat /root/.ssh/id_rsa.pub)
 162 sed -i "s|\$ssh_key|${key}|" virtlet_test_vm.yaml
 163 kubectl create -f virtlet_test_vm.yaml
 164
 165 status=""
 166 while [[ $status != "Running" ]]
 167 do
 168         stats=$(kubectl get pods |grep -i virtlet-deployment)
 169         status=$(echo $stats | cut -d " " -f 3)
 170         if [[ $status == "Err"* ]]; then
 171                 echo "Error creating Virtlet VM, test incomplete"
 172                 kubectl delete -f virtlet_test_vm.yaml
 173                 exit 1
 174         fi
 175 done
 176
 177 sleep 3
 178 echo "Virtlet VM is ready for provisioning"
 179
 180 printf "\nkubectl get pods $(kubectl get pods |grep -i virtlet-deployment | awk '{print $1}') -o json\n"
 181 podjson=$(kubectl get pods $(kubectl get pods |grep -i virtlet-deployment | awk '{print $1}') -o json)
 182 printf "\n$podjson\n\n"
 183
 184 # create provisioning cr
 185 kubectl apply -f e2e_bpa_test.yaml
 186 popd
 187
 188 sleep 2m
 189
 190 status="Running"
 191
 192 while [[ $status == "Running" ]]
 193 do
 194         stats=$(kubectl get pods |grep -i kud-cluster-vm)
 195         status=$(echo $stats | cut -d " " -f 3)
 196         echo "KUD install job still running"
 197         sleep 2m
 198 done
 199
 200 if [[ $status == "Completed" ]]; then
 201    printf "KUD Install completed successfully\n"
 202 else
 203    printf "KUD Install failed\n"
 204 fi
 205
 206 printf "\nPrinting kud-cluster-vm job logs....\n\n"
 207 kudjob=$(kubectl get pods | grep -i kud-cluster-vm | awk '{print $1}')
 208 printf "$(kubectl logs $kudjob)\n"
 209
 210 printf "\n\nBeginning E2E VM Test Teardown\n\n"
 211
 212 kubectl delete -f e2e_bpa_test.yaml
 213 kubectl delete job kud-vmcluster110
 214 kubectl delete --ignore-not-found=true configmap vmcluster110-configmap
 215 kubectl delete -f virtlet_test_vm.yaml
 216 rm -rf /opt/kud/multi-cluster/vmcluster110
 217 rm -rf $BPA_DIR