From: Todd Malsbary <todd.malsbary@intel.com>
Date: Mon, 13 Dec 2021 21:33:47 +0000 (-0800)
Subject: Add migration tool to assist with R5 to R6
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=818ebd97b869309fefa0cc102ab0a9f067f9a51d;p=icn.git

Add migration tool to assist with R5 to R6

Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I6bbf87bfd1bc4feedb73ae21cde98c833607a373
---

diff --git a/README.md b/README.md
index 72d059e..9ca60b4 100644
--- a/README.md
+++ b/README.md
@@ -192,6 +192,10 @@ provided with the values for each server. Refer to the machine chart
 in icn/deploy/machine for more details. In the example below, no DHCP
 server is present in the baremetal network.
 
+> *NOTE:* To assist in the migration of R5 and earlier release's use
+> from `nodes.json` and the Provisioning resource to a site YAML, a
+> helper script is provided at `tools/migration/to_r6.sh`.
+
 `site.yaml`
 ``` yaml
 apiVersion: v1
diff --git a/deploy/metal3/scripts/01_metal3.sh b/deploy/metal3/scripts/01_metal3.sh
index 5780858..0b0d564 100755
--- a/deploy/metal3/scripts/01_metal3.sh
+++ b/deploy/metal3/scripts/01_metal3.sh
@@ -21,60 +21,9 @@ function deprovision_compute_node {
     fi
 }
 
-function create_userdata {
-    name="$1"
-    username="$2"
-    password="$3"
-    COMPUTE_NODE_FQDN="$name.akraino.icn.org"
-
-    # validate that the user isn't expecting the deprecated
-    # COMPUTE_NODE_PASSWORD to be used
-    if [ "$password" != "${COMPUTE_NODE_PASSWORD:-$password}" ]; then
-        cat <<EOF
-COMPUTE_NODE_PASSWORD "$COMPUTE_NODE_PASSWORD" not equal to nodes.json $name password "$password".
-Unset COMPUTE_NODE_PASSWORD and retry.
-EOF
-        exit 1
-    fi
-
-    printf "userData:\n" >>${SCRIPTDIR}/${name}-values.yaml
-    if [ -n "$username" ]; then
-	printf "  name: ${username}\n" >>${SCRIPTDIR}/${name}-values.yaml
-    fi
-    if [ -n "$password" ]; then
-        passwd=$(mkpasswd --method=SHA-512 --rounds 4096 "$password")
-        printf "  hashedPassword: ${passwd}\n" >>${SCRIPTDIR}/${name}-values.yaml
-    fi
-
-    if [ -n "$COMPUTE_NODE_FQDN" ]; then
-        printf "  fqdn: ${COMPUTE_NODE_FQDN}\n" >>${SCRIPTDIR}/${name}-values.yaml
-    fi
-
-    if [ ! -f $HOME/.ssh/id_rsa.pub ]; then
-        yes y | ssh-keygen -t rsa -N "" -f $HOME/.ssh/id_rsa
-    fi
-
-    printf "  sshAuthorizedKey: $(cat $HOME/.ssh/id_rsa.pub)\n" >>${SCRIPTDIR}/${name}-values.yaml
-}
-
-create_networkdata() {
-    name="$1"
-    node_networkdata $name >>${SCRIPTDIR}/${name}-values.yaml
-}
-
 function make_bm_hosts {
     while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do
-        printf "machineName: ${name}\n" >${SCRIPTDIR}/${name}-values.yaml
-        printf "bmcUsername: ${ipmi_username}\n" >>${SCRIPTDIR}/${name}-values.yaml
-        printf "bmcPassword: ${ipmi_password}\n" >>${SCRIPTDIR}/${name}-values.yaml
-        printf "bmcAddress: ipmi://${ipmi_address}\n" >>${SCRIPTDIR}/${name}-values.yaml
-	if [[ ! -z ${boot_mac} ]]; then
-            printf "bootMACAddress: ${boot_mac}\n" >>${SCRIPTDIR}/${name}-values.yaml
-	fi
-        printf "imageName: ${BM_IMAGE}\n" >>${SCRIPTDIR}/${name}-values.yaml
-        create_userdata $name $os_username $os_password
-        create_networkdata $name
-
+	node_machine_values >${SCRIPTDIR}/${name}-values.yaml
 	helm -n metal3 install ${name} ${SCRIPTDIR}/../../machine --create-namespace -f ${SCRIPTDIR}/${name}-values.yaml
 
     done
diff --git a/env/lib/common.sh b/env/lib/common.sh
index a6a6c4b..7a91208 100755
--- a/env/lib/common.sh
+++ b/env/lib/common.sh
@@ -122,6 +122,42 @@ function list_nodes {
     fi
 }
 
+function node_userdata {
+    name="$1"
+    username="$2"
+    password="$3"
+    COMPUTE_NODE_FQDN="$name.akraino.icn.org"
+
+    # validate that the user isn't expecting the deprecated
+    # COMPUTE_NODE_PASSWORD to be used
+    if [ "$password" != "${COMPUTE_NODE_PASSWORD:-$password}" ]; then
+        cat <<EOF
+COMPUTE_NODE_PASSWORD "$COMPUTE_NODE_PASSWORD" not equal to nodes.json $name password "$password".
+Unset COMPUTE_NODE_PASSWORD and retry.
+EOF
+        exit 1
+    fi
+
+    printf "userData:\n"
+    if [ -n "$username" ]; then
+	printf "  name: ${username}\n"
+    fi
+    if [ -n "$password" ]; then
+        passwd=$(mkpasswd --method=SHA-512 --rounds 4096 "$password")
+        printf "  hashedPassword: ${passwd}\n"
+    fi
+
+    if [ -n "$COMPUTE_NODE_FQDN" ]; then
+        printf "  fqdn: ${COMPUTE_NODE_FQDN}\n"
+    fi
+
+    if [ ! -f $HOME/.ssh/id_rsa.pub ]; then
+        yes y | ssh-keygen -t rsa -N "" -f $HOME/.ssh/id_rsa
+    fi
+
+    printf "  sshAuthorizedKey: $(cat $HOME/.ssh/id_rsa.pub)\n"
+}
+
 # Returns "null" when the field is not present
 function networkdata_networks_field {
     name=$1
@@ -185,6 +221,24 @@ function wait_for {
     done
 }
 
+# This is intended to be used together with list_nodes in the
+# following way:
+#   list_nodes | while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do ...
+function node_machine_values {
+    printf "machineName: ${name}\n"
+    printf "machineLabels:\n"
+    printf "  machine: ${name}\n"
+    printf "bmcUsername: ${ipmi_username}\n"
+    printf "bmcPassword: ${ipmi_password}\n"
+    printf "bmcAddress: ipmi://${ipmi_address}\n"
+    if [[ ! -z ${boot_mac} ]]; then
+        printf "bootMACAddress: ${boot_mac}\n"
+    fi
+    printf "imageName: ${BM_IMAGE}\n"
+    node_userdata ${name} ${os_username} ${os_password}
+    node_networkdata ${name}
+}
+
 function clone_repository {
     local -r path=$1
     local -r repo=$2
diff --git a/tools/migration/to_r6.sh b/tools/migration/to_r6.sh
new file mode 100755
index 0000000..29470dd
--- /dev/null
+++ b/tools/migration/to_r6.sh
@@ -0,0 +1,171 @@
+#!/bin/bash
+set -eu -o pipefail
+
+SCRIPTDIR="$(readlink -f $(dirname ${BASH_SOURCE[0]}))"
+LIBDIR="${SCRIPTDIR}/../../env/lib"
+
+source ${LIBDIR}/common.sh
+
+function usage {
+    cat <<EOF
+Usage: $(basename $0) -n nodes.json -p provisioning.yaml >site.yaml
+
+This tool assists in migrating ICN R5 and earlier configurations to R6
+by translating an existing nodes.json and Provisioning resource YAML
+into values files to provide to the ICN machine and cluster Helm
+charts.
+
+IMPORTANT: The tool is only intended to be a starting point.  The
+following limitations should be noted:
+- The Kubernetes control plane endpoint must be explicitly specified
+  with the controlPlaneEndpoint and controlPlanePrefix values in the
+  cluster values YAML.
+- The value of image_name in nodes.json is ignored.
+- The SSH authorized key that will copied to the provisioned nodes is
+  ${HOME}/.ssh/id_rsa.pub.
+- spec.KUDPlugins in the Provisioning resource is ignored.  This
+  functionality is accomplished in R6 with Flux.
+
+After reviewing and updating the migrated site YAML as needed, the
+YAML secrets may be encrypted with the below command before committing
+to source control for use with Flux:
+
+  $(readlink -f ${SCRIPTDIR}/../../deploy/site/site.sh) sops-encrypt-site site.yaml key-name
+
+EOF
+    exit 1
+}
+
+function migrate {
+    cat <<EOF
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metal3
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: icn
+  namespace: metal3
+spec:
+  gitImplementation: go-git
+  interval: 1m0s
+  ref:
+    branch: master
+  timeout: 20s
+  url: https://gerrit.akraino.org/r/icn
+EOF
+    list_nodes | while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do
+	cat <<EOF
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: ${name}
+  namespace: metal3
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: deploy/machine
+      sourceRef:
+        kind: GitRepository
+        name: icn
+      interval: 1m
+  values:
+EOF
+	node_machine_values | sed -e 's/^/    /'
+    done
+    cat <<EOF
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: $(provisioning_json | jq -r '.metadata.name')
+  namespace: metal3
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: deploy/cluster
+      sourceRef:
+        kind: GitRepository
+        name: icn
+      interval: 1m
+    values:
+EOF
+    cluster_values | sed -e 's/^/    /'
+}
+
+function cluster_values {
+    cat <<EOF
+clusterName: $(cluster_name)
+clusterLabels:
+  owner: $(provisioning_json | jq -r '.metadata.labels.owner')
+  provider: $(provisioning_json | jq -r '.metadata.labels.owner')
+EOF
+    if [[ $(cluster_type) != "null" ]]; then
+	cat <<EOF
+  cluster-type: $(cluster_type)
+EOF
+    fi
+    cat <<EOF
+numControlPlaneMachines: $(provisioning_json | jq -r '.spec.masters | length')
+numWorkerMachines: $(provisioning_json | jq -r '.spec.workers | length')
+controlPlaneEndpoint: # TODO
+controlPlanePrefix: # TODO
+controlPlaneHostSelector:
+  matchExpressions:
+    key: machine
+    operator: In
+    values:
+$(provisioning_json | jq -r '.spec.masters[] | keys[0]' | awk '{print "    - " $0}')
+workersHostSelector:
+  matchExpressions:
+    key: machine
+    operator: In
+    values:
+$(provisioning_json | jq -r '.spec.workers[] | keys[0]' | awk '{print "    - " $0}')
+userData: {}
+# TODO
+#flux:
+#  url: https://gerrit.akraino.org/r/icn
+#  branch: master
+#  path: ./deploy/site/cluster-e2etest
+EOF
+}
+
+function cluster_name {
+    provisioning_json | jq -r '.metadata.labels.cluster'
+}
+
+function cluster_type {
+    provisioning_json | jq -r '.metadata.labels."cluster-type"'
+}
+
+function provisioning_json {
+    cat ${PROVISIONING_YAML} | python3 -c 'import sys, yaml, json; print(json.dumps(yaml.safe_load(sys.stdin.read())))'
+}
+
+while getopts ":n:o:p:" opt; do
+    case "${opt}" in
+        n)
+            NODES_JSON=${OPTARG}
+            ;;
+        p)
+            PROVISIONING_YAML=${OPTARG}
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [[ -z "${NODES_JSON}" ]] || [[ -z "${PROVISIONING_YAML}" ]]; then
+    usage
+fi
+
+export NODES_FILE=${NODES_JSON}
+migrate