From e3be88235a3456ea1b7fd34460d21d894ba21871 Mon Sep 17 00:00:00 2001
From: Todd Malsbary <todd.malsbary@intel.com>
Date: Fri, 24 Sep 2021 15:07:14 -0700
Subject: [PATCH] Use Vagrantfile to build virtual site

Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I9498be3425877ed21b48c1499c0c9db969be6185
---
 Vagrantfile                                   | 154 ++++++++++++++++++++++++--
 tools/vagrant/add_machine_to_vbmc.sh          |   9 ++
 tools/vagrant/create_nodes_json_sample.sh     |  67 +++++++++++
 tools/vagrant/create_provisioning_cr.sh       |  41 +++++++
 tools/vagrant/create_provisioning_network.sh  |  17 +++
 tools/vagrant/create_user_config.sh           |  12 ++
 tools/vagrant/destroy_provisioning_network.sh |   9 ++
 tools/vagrant/remove_machine_from_vbmc.sh     |   8 ++
 tools/vagrant/start_vbmc.sh                   |  22 ++++
 tools/vagrant/stop_vbmc.sh                    |   7 ++
 10 files changed, 336 insertions(+), 10 deletions(-)
 create mode 100755 tools/vagrant/add_machine_to_vbmc.sh
 create mode 100755 tools/vagrant/create_nodes_json_sample.sh
 create mode 100755 tools/vagrant/create_provisioning_cr.sh
 create mode 100755 tools/vagrant/create_provisioning_network.sh
 create mode 100755 tools/vagrant/create_user_config.sh
 create mode 100755 tools/vagrant/destroy_provisioning_network.sh
 create mode 100755 tools/vagrant/remove_machine_from_vbmc.sh
 create mode 100755 tools/vagrant/start_vbmc.sh
 create mode 100755 tools/vagrant/stop_vbmc.sh

diff --git a/Vagrantfile b/Vagrantfile
index 25d0ad4..f3c700e 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -1,16 +1,150 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
+# IMPORTANT To bring up the machines, use the "--no-parallel" option
+# to vagrant up.  This is to workaround dependencies between the jump
+# machine and the machine pool machines.  Specifically, the pool
+# machines will fail to come up until the baremetal network (created
+# by vagrant from the jump machine definition) is up.
+
+vars = {
+  :site => 'vm',
+  :baremetal_cidr => '192.168.151.0/24',
+  :num_machines => 2
+}
+
+$post_up_message = <<MSG
+------------------------------------------------------
+
+To get started with ICN:
+
+  $ vagrant ssh jump
+  vagrant@jump:~$ sudo su
+  root@jump:/home/vagrant# cd /icn
+  root@jump:/home/vagrant# make install
+
+------------------------------------------------------
+MSG
+
+#
+# Networks
+#
+# The ICN baremetal network will be the vagrant management network.
+# It is created automatically by vagrant.  The provisioning network
+# will be a vagrant private network, and is required to be created by
+# this script.  The IPMI network is created with virtualbmc.
+
+#
+# Machines
+#
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/ubuntu1804"
-  config.vm.hostname = "ubuntu18"
-  config.vm.synced_folder ".", "/vagrant"
-  config.vm.provider :libvirt do |libvirt|
-    libvirt.graphics_ip = '0.0.0.0'
-    # add random suffix to allow running multiple jobs
-    libvirt.random_hostname = 'yes'
-    libvirt.cpus = 32
-    libvirt.memory = 40960
-    libvirt.machine_virtual_size = 400
+  # The jump machine
+  config.vm.define 'jump' do |m|
+    # Note the apparent typo in the name below, it is correct as-is
+    m.vm.box = 'intergratedcloudnative/ubuntu1804'
+    m.vm.hostname = 'jump'
+    m.vm.synced_folder '.', '/icn'
+    m.vm.provider :libvirt do |libvirt|
+      libvirt.graphics_ip = '0.0.0.0'
+      libvirt.default_prefix = "#{vars[:site]}-"
+      libvirt.cpu_mode = 'host-passthrough'
+      libvirt.cpus = 8
+      libvirt.memory = 16384
+      libvirt.nested = true
+
+      # The ICN baremetal network is the vagrant management network,
+      # and is created by vagrant for us
+      libvirt.management_network_name = "#{vars[:site]}-baremetal"
+      libvirt.management_network_address = vars[:baremetal_cidr]
+      libvirt.management_network_autostart = true
+    end
+
+    # The ICN provisioning network will be a vagrant private network
+    # created upon bringing up the jump machine
+    m.trigger.before [:up] do |trigger|
+      trigger.name = 'Creating provisioning network'
+      trigger.run = {inline: "./tools/vagrant/create_provisioning_network.sh #{vars[:site]}"}
+    end
+    m.trigger.after [:destroy] do |trigger|
+      trigger.name = 'Destroying provisioning network'
+      trigger.run = {inline: "./tools/vagrant/destroy_provisioning_network.sh #{vars[:site]}"}
+    end
+    m.vm.network :private_network,
+                 :libvirt__network_name => "#{vars[:site]}-provisioning",
+                 :type => 'dhcp'
+
+    # IPMI control of machines is provided by vbmc on the host
+    m.trigger.after [:up] do |trigger|
+      trigger.name = 'Starting virtualbmc for IPMI network'
+      trigger.run = {inline: "./tools/vagrant/start_vbmc.sh"}
+    end
+    m.trigger.after [:destroy] do |trigger|
+      trigger.name = 'Stopping virtualbmc for IPMI network'
+      trigger.run = {inline: "./tools/vagrant/stop_vbmc.sh"}
+    end
+
+    m.trigger.after [:up] do |trigger|
+      trigger.name = 'Creating ICN user_config.sh'
+      trigger.run = {inline: "./tools/vagrant/create_user_config.sh"}
+    end
+    m.vm.provision 'Configuring ICN prerequisites', type: 'shell', privileged: true, inline: <<-SHELL
+      ssh-keygen -f "${HOME}/.ssh/id_rsa" -P "" <<<y
+      DEBIAN_FRONTEND=noninteractive apt-get install -y make
+    SHELL
+    m.vm.post_up_message = $post_up_message
+  end
+
+  # The machine pool used by cluster creation
+  (1..vars[:num_machines]).each do |i|
+    config.vm.define "machine-#{i}" do |m|
+      m.vm.hostname = "machine-#{i}"
+      m.vm.provider :libvirt do |libvirt|
+        libvirt.graphics_ip = '0.0.0.0'
+        libvirt.default_prefix = "#{vars[:site]}-"
+        libvirt.cpu_mode = 'host-passthrough'
+        libvirt.cpus = 8
+        libvirt.memory = 16384
+        libvirt.nested = true
+        # The image will be provisioned by ICN so just create an empty
+        # disk for the machine
+        libvirt.storage :file, :size => 50, :type => 'raw', :cache => 'none'
+        # Management attach is false so that vagrant will not interfere
+        # with these machines: the jump server will manage them
+        # completely
+        libvirt.mgmt_attach = false
+      end
+      # The provisioning network must be listed first for PXE boot to
+      # the metal3/ironic provided image
+      m.vm.network :private_network,
+                   :libvirt__network_name => "#{vars[:site]}-provisioning",
+                   :type => 'dhcp'
+      m.vm.network :private_network,
+                   :libvirt__network_name => "#{vars[:site]}-baremetal",
+                   :type => 'dhcp'
+
+      # IPMI control
+      m.trigger.after [:up] do |trigger|
+        trigger.name = 'Adding machine to IPMI network'
+        trigger.run = {inline: "./tools/vagrant/add_machine_to_vbmc.sh #{i} #{vars[:site]} machine-#{i}"}
+      end
+      m.trigger.after [:destroy] do |trigger|
+        trigger.name = 'Removing machine from IPMI network'
+        trigger.run = {inline: "./tools/vagrant/remove_machine_from_vbmc.sh #{i} #{vars[:site]} machine-#{i}"}
+      end
+
+      # Create configuration for ICN provisioning
+      m.trigger.after [:up] do |trigger|
+        if i == vars[:num_machines] then
+          trigger.info = 'Creating nodes.json.sample describing the machines'
+          trigger.run = {inline: "./tools/vagrant/create_nodes_json_sample.sh #{vars[:num_machines]} #{vars[:site]} machine-"}
+        end
+      end
+      m.trigger.after [:up] do |trigger|
+        if i == vars[:num_machines] then
+          trigger.info = 'Creating Provisioning resource describing the cluster'
+          trigger.run = {inline: "./tools/vagrant/create_provisioning_cr.sh #{vars[:num_machines]} #{vars[:site]} machine-"}
+        end
+      end
+    end
   end
 end
diff --git a/tools/vagrant/add_machine_to_vbmc.sh b/tools/vagrant/add_machine_to_vbmc.sh
new file mode 100755
index 0000000..5676a7b
--- /dev/null
+++ b/tools/vagrant/add_machine_to_vbmc.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eu -o pipefail
+
+index=$1
+site=$2
+name=$3
+
+vbmc --no-daemon add ${site}-${name} --port $((6230+index-1)) --libvirt-uri "qemu:///system?&no_verify=1&no_tty=1"
+vbmc --no-daemon start ${site}-${name}
diff --git a/tools/vagrant/create_nodes_json_sample.sh b/tools/vagrant/create_nodes_json_sample.sh
new file mode 100755
index 0000000..b0ed2b4
--- /dev/null
+++ b/tools/vagrant/create_nodes_json_sample.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+set -eu -o pipefail
+
+num_machines=$1
+site=$2
+name_prefix=$3
+
+nodes_json_path="deploy/metal3/scripts/nodes.json.sample"
+ipmi_host=$(virsh -c qemu:///system net-dumpxml ${site}-baremetal | xmlstarlet sel -t -v "//network/ip/@address")
+
+cat <<EOF >${nodes_json_path}
+{
+  "nodes": [
+EOF
+for ((i=1;i<=num_machines;++i)); do
+    name="${name_prefix}${i}"
+    ipmi_port=$((6230+i-1))
+    baremetal_mac=$(virsh -c qemu:///system dumpxml "${site}-${name}" | xmlstarlet sel -t -v "//interface[source/@network='${site}-baremetal']/mac/@address")
+    provisioning_mac=$(virsh -c qemu:///system dumpxml "${site}-${name}" | xmlstarlet sel -t -v "//interface[source/@network='${site}-provisioning']/mac/@address")
+    if ((i<num_machines)); then comma=","; else comma=""; fi
+    cat <<EOF >>${nodes_json_path}
+    {
+      "name": "${name}",
+      "ipmi_driver_info": {
+        "username": "admin",
+        "password": "password",
+        "address": "${ipmi_host}:${ipmi_port}"
+      },
+      "os": {
+        "image_name": "bionic-server-cloudimg-amd64.img",
+        "username": "ubuntu",
+        "password": "mypasswd"
+      },
+      "net": {
+        "links": [
+          {
+            "id": "baremetal_nic",
+            "ethernet_mac_address": "${baremetal_mac}",
+            "type": "phy"
+          },
+          {
+            "id": "provisioning_nic",
+            "ethernet_mac_address": "${provisioning_mac}",
+            "type": "phy"
+          }
+        ],
+        "networks": [
+          {
+            "id": "baremetal",
+            "link": "baremetal_nic",
+            "type": "ipv4_dhcp"
+          },
+          {
+            "id": "provisioning",
+            "link": "provisioning_nic",
+            "type": "ipv4_dhcp"
+          }
+        ],
+        "services": []
+      }
+    }${comma}
+EOF
+done
+cat <<EOF >>${nodes_json_path}
+  ]
+}
+EOF
diff --git a/tools/vagrant/create_provisioning_cr.sh b/tools/vagrant/create_provisioning_cr.sh
new file mode 100755
index 0000000..2d07344
--- /dev/null
+++ b/tools/vagrant/create_provisioning_cr.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+set -eu -o pipefail
+
+num_machines=$1
+site=$2
+name_prefix=$3
+
+provisioning_cr_path="cmd/bpa-operator/e2etest/test_bmh_provisioning_cr.yaml"
+
+name="${name_prefix}1"
+provisioning_mac=$(virsh -c qemu:///system dumpxml "${site}-${name}" | xmlstarlet sel -t -v "//interface[source/@network='${site}-provisioning']/mac/@address")
+cat <<EOF >${provisioning_cr_path}
+apiVersion: bpa.akraino.org/v1alpha1
+kind: Provisioning
+metadata:
+  name: provisioning-test-bmh
+  labels:
+    cluster: test-bmh-cluster
+    owner: tester
+spec:
+  masters:
+    - ${name}:
+        mac-address: ${provisioning_mac}
+EOF
+if ((num_machines>1)); then
+    cat <<EOF >>${provisioning_cr_path}
+  workers:
+EOF
+    for ((i=2;i<=num_machines;++i)); do
+	name="${name_prefix}${i}"
+	provisioning_mac=$(virsh -c qemu:///system dumpxml "${site}-${name}" | xmlstarlet sel -t -v "//interface[source/@network='${site}-provisioning']/mac/@address")
+	cat <<EOF >>${provisioning_cr_path}
+    - ${name}:
+        mac-address: ${provisioning_mac}
+EOF
+    done
+fi
+cat <<EOF >>${provisioning_cr_path}
+  KUDPlugins:
+    - emco
+EOF
diff --git a/tools/vagrant/create_provisioning_network.sh b/tools/vagrant/create_provisioning_network.sh
new file mode 100755
index 0000000..c2f125c
--- /dev/null
+++ b/tools/vagrant/create_provisioning_network.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+set -eu -o pipefail
+
+site=$1
+
+if virsh -c qemu:///system net-info ${site}-provisioning >/dev/null 2>&1; then
+    echo provisioning network is already created
+else
+    cat <<EOF >${site}-provisioning-network.xml
+<network>
+  <name>${site}-provisioning</name>
+  <bridge name="${site}0"/>
+</network>
+EOF
+    virsh -c qemu:///system net-define ${site}-provisioning-network.xml
+    virsh -c qemu:///system net-start ${site}-provisioning
+fi
diff --git a/tools/vagrant/create_user_config.sh b/tools/vagrant/create_user_config.sh
new file mode 100755
index 0000000..bda1491
--- /dev/null
+++ b/tools/vagrant/create_user_config.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eu -o pipefail
+
+cat <<EOF >user_config.sh
+#!/usr/bin/env bash
+
+#Ironic Metal3 settings for provisioning network
+export IRONIC_INTERFACE="eth1"
+
+#Ironic Metal3 setting for IPMI LAN Network
+export IRONIC_IPMI_INTERFACE="eth0"
+EOF
diff --git a/tools/vagrant/destroy_provisioning_network.sh b/tools/vagrant/destroy_provisioning_network.sh
new file mode 100755
index 0000000..0759bf1
--- /dev/null
+++ b/tools/vagrant/destroy_provisioning_network.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eu -o pipefail
+
+site=$1
+
+if virsh -c qemu:///system net-info ${site}-provisioning >/dev/null 2>&1; then
+   virsh -c qemu:///system net-destroy ${site}-provisioning
+   virsh -c qemu:///system net-undefine ${site}-provisioning
+fi
diff --git a/tools/vagrant/remove_machine_from_vbmc.sh b/tools/vagrant/remove_machine_from_vbmc.sh
new file mode 100755
index 0000000..4ff8018
--- /dev/null
+++ b/tools/vagrant/remove_machine_from_vbmc.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eu -o pipefail
+
+index=$1
+site=$2
+name=$3
+
+vbmc --no-daemon delete ${site}-${name} || true
diff --git a/tools/vagrant/start_vbmc.sh b/tools/vagrant/start_vbmc.sh
new file mode 100755
index 0000000..7187a3f
--- /dev/null
+++ b/tools/vagrant/start_vbmc.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -eu -o pipefail
+
+if [[ -f ${HOME}/.vbmc/master.pid && $(ps -p $(cat ${HOME}/.vbmc/master.pid) 2>/dev/null) ]]; then
+    echo virtualbmc is already started
+else
+    if [[ $(which apt-get 2>/dev/null) ]]; then
+	DEBIAN_FRONTEND=noninteractive sudo apt-get install -y make libvirt-dev python3-pip
+    elif [[ $(which yum) ]]; then
+	sudo yum install -y make libvirt-devel python3-pip
+    fi
+    sudo python3 -m pip install libvirt-python virtualbmc
+    mkdir -p ${HOME}/.vbmc
+    cat <<EOF >${HOME}/.vbmc/virtualbmc.conf
+[log]
+logfile=${HOME}/.vbmc/virtualbmc.log
+debug=True
+[ipmi]
+session_timout=20
+EOF
+    vbmcd
+fi
diff --git a/tools/vagrant/stop_vbmc.sh b/tools/vagrant/stop_vbmc.sh
new file mode 100755
index 0000000..dec76fd
--- /dev/null
+++ b/tools/vagrant/stop_vbmc.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eu -o pipefail
+
+if [[ -f ${HOME}/.vbmc/master.pid && $(ps -p $(cat ${HOME}/.vbmc/master.pid) 2>/dev/null) ]]; then
+    kill $(cat ${HOME}/.vbmc/master.pid)
+    echo Stopped virtualbmc
+fi
-- 
2.16.6