cmd/bpa-operator/e2etest/bpa_virtletvm_verifier.sh

   1 #!/bin/bash
   2
   3 printf "\n\nStart Virtlet VM provisioning E2E test\n\n"
   4
   5 TUNING_DIR="/tmp/tuning_dir"
   6 CNI_PLUGINS="cni-plugins-linux-amd64-v0.8.2.tgz"
   7 if !(wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/$CNI_PLUGINS -P $TUNING_DIR 2>/dev/null); then
   8     echo "Error downloading cni plugins for Virtlet VM provisioning"
   9     exit 1
  10 fi
  11
  12 pushd $TUNING_DIR
  13 if [ -f $CNI_PLUGINS ]; then
  14     tar -xzvf $CNI_PLUGINS > /dev/null
  15     if [ -f "tuning" ]; then
  16         cp "tuning" "/opt/cni/bin/"
  17         echo "Updated the tuning plugin"
  18     else
  19         echo "Error finding the latest tuning plugin"
  20         rm -rf $TUNING_DIR
  21         exit 1
  22     fi
  23     rm -rf $TUNING_DIR
  24 fi
  25 popd
  26
  27 # Create network attachment definition
  28 BPA_DIR="/tmp/bpa"
  29 mkdir -p $BPA_DIR
  30 cat <<EOF > $BPA_DIR/netattachdef-flannel-vm.yaml
  31 apiVersion: "k8s.cni.cncf.io/v1"
  32 kind: NetworkAttachmentDefinition
  33 metadata:
  34   name: flannel-vm
  35 spec:
  36   config: '{
  37             "cniVersion": "0.3.1",
  38             "name" : "cni0",
  39             "plugins": [ {
  40               "type": "flannel",
  41               "cniVersion": "0.3.1",
  42               "masterplugin": true,
  43               "delegate": {
  44                   "isDefaultGateway": true
  45               }
  46             },
  47             {
  48               "type": "tuning"
  49             }]
  50           }'
  51 EOF
  52
  53 cat <<'EOF' > $BPA_DIR/virtlet_test_vm.yaml
  54 apiVersion: apps/v1
  55 kind: Deployment
  56 metadata:
  57   name: virtlet-deployment
  58   labels:
  59     app: virtlet
  60 spec:
  61   replicas: 1
  62   selector:
  63     matchLabels:
  64       app: virtlet
  65   template:
  66     metadata:
  67       labels:
  68         app: virtlet
  69       annotations:
  70         VirtletLibvirtCPUSetting: |
  71           mode: host-passthrough
  72         # This tells CRI Proxy that this pod belongs to Virtlet runtime
  73         kubernetes.io/target-runtime: virtlet.cloud
  74         VirtletCloudInitUserData: |
  75           ssh_pwauth: True
  76           disable_root: false
  77           chpasswd: {expire: False}
  78           manage_resolv_conf: True
  79           resolv_conf:
  80             nameservers: ['8.8.8.8', '8.8.4.4']
  81           users:
  82           - name: root
  83             gecos: User
  84             primary-group: root
  85             groups: users
  86             lock_passwd: false
  87             shell: /bin/bash
  88             sudo: ALL=(ALL) NOPASSWD:ALL
  89             ssh_authorized_keys:
  90               $ssh_key
  91           runcmd:
  92             - sed -i -e 's/^#DNS=.*/DNS=8.8.8.8/g' /etc/systemd/resolved.conf
  93             - systemctl daemon-reload
  94             - systemctl restart systemd-resolved
  95         v1.multus-cni.io/default-network: '[
  96             { "name": "flannel-vm",
  97               "mac": "c2:b4:57:49:47:f1" }]'
  98         VirtletRootVolumeSize: 8Gi
  99         VirtletVCPUCount: "2"
 100     spec:
 101       affinity:
 102         nodeAffinity:
 103           requiredDuringSchedulingIgnoredDuringExecution:
 104             nodeSelectorTerms:
 105             - matchExpressions:
 106               - key: extraRuntime
 107                 operator: In
 108                 values:
 109                 - virtlet
 110       containers:
 111       - name: virtlet-deployment
 112         # This specifies the image to use.
 113         # virtlet.cloud/ prefix is used by CRI proxy, the remaining part
 114         # of the image name is prepended with https:// and used to download the image
 115         image: virtlet.cloud/ubuntu/18.04
 116         imagePullPolicy: IfNotPresent
 117         # tty and stdin required for "kubectl attach -t" to work
 118         tty: true
 119         stdin: true
 120         resources:
 121           requests:
 122             cpu: 2
 123             memory: 8Gi
 124           limits:
 125             # This memory limit is applied to the libvirt domain definition
 126             cpu: 2
 127             memory: 8Gi
 128 EOF
 129
 130 # Create provisioning CR file for BPA testing
 131 cat <<EOF > $BPA_DIR/e2e_bpa_test.yaml
 132 apiVersion: bpa.akraino.org/v1alpha1
 133 kind: Provisioning
 134 metadata:
 135   name: vmcluster110
 136   labels:
 137     cluster: vmcluster110
 138     cluster-type: virtlet-vm
 139     owner: c1
 140 spec:
 141   masters:
 142     - master-1:
 143         mac-address: c2:b4:57:49:47:f1
 144   PodSubnet: 172.21.64.0/18
 145 EOF
 146
 147 pushd $BPA_DIR
 148 # create flannel-vm net-attach-def
 149 kubectl apply -f netattachdef-flannel-vm.yaml -n kube-system
 150
 151 # generate user ssh key
 152 if [ ! -f "/root/.ssh/id_rsa.pub" ]; then
 153     ssh-keygen -f /root/.ssh/id_rsa -P ""
 154 fi
 155
 156 # create ssh key secret
 157 kubectl create secret generic ssh-key-secret --from-file=id_rsa=/root/.ssh/id_rsa --from-file=id_rsa.pub=/root/.ssh/id_rsa.pub
 158
 159 # create virtlet vm
 160 key=$(cat /root/.ssh/id_rsa.pub)
 161 sed -i "s|\$ssh_key|${key}|" virtlet_test_vm.yaml
 162 kubectl create -f virtlet_test_vm.yaml
 163
 164 status=""
 165 while [[ $status != "Running" ]]
 166 do
 167         stats=$(kubectl get pods |grep -i virtlet-deployment)
 168         status=$(echo $stats | cut -d " " -f 3)
 169         if [[ $status == "Err"* ]]; then
 170                 echo "Error creating Virtlet VM, test incomplete"
 171                 kubectl delete -f virtlet_test_vm.yaml
 172                 exit 1
 173         fi
 174 done
 175
 176 sleep 3
 177 echo "Virtlet VM is ready for provisioning"
 178
 179 printf "\nkubectl get pods $(kubectl get pods |grep -i virtlet-deployment | awk '{print $1}') -o json\n"
 180 podjson=$(kubectl get pods $(kubectl get pods |grep -i virtlet-deployment | awk '{print $1}') -o json)
 181 printf "\n$podjson\n\n"
 182
 183 # create provisioning cr
 184 kubectl apply -f e2e_bpa_test.yaml
 185 popd
 186
 187 sleep 2m
 188
 189 status="Running"
 190
 191 while [[ $status == "Running" ]]
 192 do
 193         stats=$(kubectl get pods |grep -i kud-cluster-vm)
 194         status=$(echo $stats | cut -d " " -f 3)
 195         echo "KUD install job still running"
 196         sleep 2m
 197 done
 198
 199 if [[ $status == "Completed" ]]; then
 200    printf "KUD Install completed successfully\n"
 201 else
 202    printf "KUD Install failed\n"
 203 fi
 204
 205 printf "\nPrinting kud-cluster-vm job logs....\n\n"
 206 kudjob=$(kubectl get pods | grep -i kud-cluster-vm | awk '{print $1}')
 207 printf "$(kubectl logs $kudjob)\n"
 208
 209 printf "\n\nBeginning E2E VM Test Teardown\n\n"
 210
 211 kubectl delete -f e2e_bpa_test.yaml
 212 kubectl delete job kud-vmcluster110
 213 kubectl delete configmap vmcluster110-configmap
 214 kubectl delete -f virtlet_test_vm.yaml
 215 rm -rf /opt/kud/multi-cluster/vmcluster110
 216 rm -rf $BPA_DIR