From 164e86c74cbf20cbab255f355c8021bf6a4d729d Mon Sep 17 00:00:00 2001 From: Todd Malsbary Date: Thu, 28 Oct 2021 16:07:56 -0700 Subject: [PATCH] Capture creation of BMH resources into Helm chart Signed-off-by: Todd Malsbary Change-Id: Ie67048142020a0eb4aa544ddba9f551fbd1eec72 --- .gitignore | 1 + deploy/machines/.helmignore | 23 ++++ deploy/machines/Chart.yaml | 4 + deploy/machines/example-values.yaml | 27 +++++ deploy/machines/templates/_networkdata.json | 40 +++++++ deploy/machines/templates/_userdata.yaml | 48 ++++++++ deploy/machines/templates/baremetalhost.yaml | 61 ++++++++++ deploy/metal3/scripts/01_metal3.sh | 167 +++++---------------------- env/lib/common.sh | 43 ++++++- 9 files changed, 273 insertions(+), 141 deletions(-) create mode 100644 deploy/machines/.helmignore create mode 100644 deploy/machines/Chart.yaml create mode 100644 deploy/machines/example-values.yaml create mode 100644 deploy/machines/templates/_networkdata.json create mode 100644 deploy/machines/templates/_userdata.yaml create mode 100644 deploy/machines/templates/baremetalhost.yaml diff --git a/.gitignore b/.gitignore index 11e35ac..371c05f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ deploy/ironic/logs/ deploy/baremetal-operator/logs/ deploy/cert-manager/logs/ +deploy/metal3/scripts/machines-values.yaml diff --git a/deploy/machines/.helmignore b/deploy/machines/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deploy/machines/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/machines/Chart.yaml b/deploy/machines/Chart.yaml new file mode 100644 index 0000000..aa232fd --- /dev/null +++ b/deploy/machines/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: machines +type: application +version: 0.1.0 diff --git a/deploy/machines/example-values.yaml b/deploy/machines/example-values.yaml new file mode 100644 index 0000000..7d6b16c --- /dev/null +++ b/deploy/machines/example-values.yaml @@ -0,0 +1,27 @@ +machines: + machine-1: + bmcUsername: admin + bmcPassword: password + bmcAddress: ipmi://192.168.151.1:6230 + + # Optional + bootMACAddress: 52:54:00:2b:bc:3a + + # Optional + imageName: bionic-server-cloudimg-amd64.img + + # Optional + networks: + baremetal: + macAddress: 52:54:00:da:c9:7b + type: ipv4_dhcp + provisioning: + macAddress: 52:54:00:2b:bc:3a + type: ipv4_dhcp + + # Optional + userData: + name: ubuntu + hashedPassword: $6$rounds=4096$acxyX2VqfHJSAc2$sgVf5uTHHPCX6u50NHnJmhIoqbcL9J12jlBAaWKvd3w8uYO0iXgcBrEhtvHLgSGU7dcU.eqm9JwXEYbbRjPAi1 + sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump + fqdn: machine-1.akraino.icn.org diff --git a/deploy/machines/templates/_networkdata.json b/deploy/machines/templates/_networkdata.json new file mode 100644 index 0000000..ea407cc --- /dev/null +++ b/deploy/machines/templates/_networkdata.json @@ -0,0 +1,40 @@ +{{- define "machines.networkData" -}} +{{- if .networks -}} +{ + "links": [ +{{- $local := dict "first" true -}} +{{- range $name, $network := .networks }} +{{- if not $local.first }} + },{ +{{- else }} + { +{{- end }} +{{- $_ := set $local "first" false }} + "id": "{{ $name }}_nic", + "ethernet_mac_address": "{{ $network.macAddress }}", + "type": "phy" +{{- end }} + } + ], + "networks": [ +{{- $local := dict "first" true -}} +{{- range $name, $network := .networks }} +{{- if not $local.first }} + },{ +{{- else }} + { +{{- end }} +{{- $_ := set $local "first" false }} + "id": "{{ $name }}", + "link": "{{ $name }}_nic", + "type": "{{ $network.type }}"{{- if $network.ipAddress }}, + "ip_address": "{{ $network.ipAddress }}"{{- end }}{{- if $network.gateway }}, + "gateway": "{{ $network.gateway }}"{{- end }}{{- if $network.nameservers }}, + "dns_nameservers": {{ $network.nameservers }}{{- end }} +{{- end }} + } + ], + "services": [] +} +{{ end }} +{{- end }} diff --git a/deploy/machines/templates/_userdata.yaml b/deploy/machines/templates/_userdata.yaml new file mode 100644 index 0000000..5f76ef6 --- /dev/null +++ b/deploy/machines/templates/_userdata.yaml @@ -0,0 +1,48 @@ +{{- define "machines.userData" -}} +{{- if .userData -}} +#cloud-config +{{- if and .userData.name .userData.hashedPassword }} +users: +- name: {{ .userData.name }} + lock_passwd: False + passwd: {{ .userData.hashedPassword }} + sudo: "ALL=(ALL) NOPASSWD:ALL" +{{- else if .userData.hashedPassword }} +password: {{ .userData.hashedPassword }} +{{- end }} +chpasswd: {expire: False} +ssh_pwauth: True +fqdn: {{ .userData.fqdn }} +disable_root: false +ssh_authorized_keys: +- {{ .userData.sshAuthorizedKey }} +write_files: +- path: /var/lib/cloud/scripts/per-instance/set_dhcp_identifier.sh + # The IP address assigned to the provisioning NIC will change due to + # IPA using the MAC address as the client ID and systemd using a + # different ID. Tell systemd to use the MAC as the client ID. We + # can't do this in the network data as only the JSON format is + # supported by metal3, and the JSON format does not support the + # dhcp-identifier field. + owner: root:root + permissions: '0777' + content: | + #!/usr/bin/env bash + set -eux -o pipefail + sed -i -e '/dhcp4: true$/!b' -e 'h;s/\S.*/dhcp-identifier: mac/;H;g' /etc/netplan/50-cloud-init.yaml + netplan apply +- path: /var/lib/cloud/scripts/per-instance/set_kernel_cmdline.sh + # The "intel_iommu=on iommu=pt" kernel command line is necessary for + # QAT support. + owner: root:root + permissions: '0777' + content: | + #!/usr/bin/env bash + set -eux -o pipefail + grub_file=${1:-"/etc/default/grub"} + kernel_parameters="intel_iommu=on iommu=pt" + sed -i~ "/^GRUB_CMDLINE_LINUX=/{h;s/\(=\".*\)\"/\1 ${kernel_parameters}\"/};\${x;/^$/{s//GRUB_CMDLINE_LINUX=\"${kernel_parameters}\"/;H};x}" "$grub_file" + update-grub + reboot +{{ end }} +{{- end }} diff --git a/deploy/machines/templates/baremetalhost.yaml b/deploy/machines/templates/baremetalhost.yaml new file mode 100644 index 0000000..961380c --- /dev/null +++ b/deploy/machines/templates/baremetalhost.yaml @@ -0,0 +1,61 @@ +{{- range $name, $machine := .Values.machines }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-bmc-secret +type: Opaque +data: + username: {{ $machine.bmcUsername | b64enc }} + password: {{ $machine.bmcPassword | b64enc }} +{{- if $machine.networks }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-network-data +type: Opaque +data: + networkData: {{ include "machines.networkData" $machine | b64enc }} +{{- end }} +{{- if $machine.userData }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-user-data +type: Opaque +data: + userData: {{ include "machines.userData" $machine | b64enc }} +{{- end }} +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: {{ $name }} +spec: + online: true +{{- if $machine.bootMACAddress }} + bootMACAddress: {{ $machine.bootMACAddress }} +{{- end }} + bmc: + address: {{ $machine.bmcAddress }} + credentialsName: {{ $name }}-bmc-secret +{{- if $machine.imageName }} + image: + url: http://172.22.0.1:6180/images/{{ $machine.imageName }} + checksum: http://172.22.0.1:6180/images/{{ $machine.imageName }}.md5sum +{{- end }} +{{- if $machine.networks }} + networkData: + name: {{ $name }}-network-data + namespace: {{ $.Release.Namespace }} +{{- end }} +{{- if $machine.userData }} + userData: + name: {{ $name }}-user-data + namespace: {{ $.Release.Namespace }} +{{- end }} + rootDeviceHints: + minSizeGigabytes: 48 +{{- end }} diff --git a/deploy/metal3/scripts/01_metal3.sh b/deploy/metal3/scripts/01_metal3.sh index 8d0fc1b..add2a29 100755 --- a/deploy/metal3/scripts/01_metal3.sh +++ b/deploy/metal3/scripts/01_metal3.sh @@ -1,20 +1,18 @@ #!/usr/bin/env bash set -eu -o pipefail -LIBDIR="$(dirname "$(dirname "$(dirname "$PWD")")")" +SCRIPTDIR="$(readlink -f $(dirname ${BASH_SOURCE[0]}))" +LIBDIR="$(dirname $(dirname $(dirname ${SCRIPTDIR})))/env/lib" eval "$(go env)" -source $LIBDIR/env/lib/common.sh +source $LIBDIR/common.sh if [[ $EUID -ne 0 ]]; then echo "This script must be run as root" exit 1 fi -IMAGE_URL=http://172.22.0.1:6180/images/${BM_IMAGE} -IMAGE_CHECKSUM=http://172.22.0.1:6180/images/${BM_IMAGE}.md5sum - function deprovision_compute_node { name="$1" if kubectl get baremetalhost $name -n metal3 &>/dev/null; then @@ -23,8 +21,6 @@ function deprovision_compute_node { fi } -# documentation for the values below may be found at -# https://cloudinit.readthedocs.io/en/latest/topics/modules.html function create_userdata { name="$1" username="$2" @@ -41,136 +37,43 @@ EOF exit 1 fi - printf "#cloud-config\n" > $name-userdata.yaml + printf " userData:\n" >>${SCRIPTDIR}/machines-values.yaml + if [ -n "$username" ]; then + printf " name: ${username}\n" >>${SCRIPTDIR}/machines-values.yaml + fi if [ -n "$password" ]; then - if [ -n "$username" ]; then - passwd=$(mkpasswd --method=SHA-512 --rounds 4096 "$password") - printf "users:" >> $name-userdata.yaml - printf "\n - name: ""%s" "$username" >> $name-userdata.yaml - printf "\n lock_passwd: False" >> $name-userdata.yaml # necessary to allow password login - printf "\n passwd: ""%s" "$passwd" >> $name-userdata.yaml - printf "\n sudo: \"ALL=(ALL) NOPASSWD:ALL\"" >> $name-userdata.yaml - else - printf "password: ""%s" "$password" >> $name-userdata.yaml - fi - printf "\nchpasswd: {expire: False}\n" >> $name-userdata.yaml - printf "ssh_pwauth: True\n" >> $name-userdata.yaml + passwd=$(mkpasswd --method=SHA-512 --rounds 4096 "$password") + printf " hashedPassword: ${passwd}\n" >>${SCRIPTDIR}/machines-values.yaml fi if [ -n "$COMPUTE_NODE_FQDN" ]; then - printf "fqdn: ""%s" "$COMPUTE_NODE_FQDN" >> $name-userdata.yaml - printf "\n" >> $name-userdata.yaml + printf " fqdn: ${COMPUTE_NODE_FQDN}\n" >>${SCRIPTDIR}/machines-values.yaml fi - printf "disable_root: false\n" >> $name-userdata.yaml - printf "ssh_authorized_keys:\n - " >> $name-userdata.yaml if [ ! -f $HOME/.ssh/id_rsa.pub ]; then yes y | ssh-keygen -t rsa -N "" -f $HOME/.ssh/id_rsa fi - cat $HOME/.ssh/id_rsa.pub >> $name-userdata.yaml - cloud_init_scripts >> $name-userdata.yaml - printf "\n" >> $name-userdata.yaml + printf " sshAuthorizedKey: $(cat $HOME/.ssh/id_rsa.pub)\n" >>${SCRIPTDIR}/machines-values.yaml } create_networkdata() { name="$1" - node_networkdata $name > $name-networkdata.json -} - -function cloud_init_scripts { - # set_dhcp_indentifier.sh: - # The IP address assigned to the provisioning NIC will change - # due to IPA using the MAC address as the client ID and systemd - # using a different ID. Tell systemd to use the MAC as the - # client ID. We can't do this in the network data as only the - # JSON format is supported by metal3, and the JSON format does - # not support the dhcp-identifier field. - # set_kernel_cmdline.sh: - # The "intel_iommu=on iommu=pt" kernel command line is necessary - # for QAT support. - cat << 'EOF' -write_files: -- path: /var/lib/cloud/scripts/per-instance/set_dhcp_identifier.sh - owner: root:root - permissions: '0777' - content: | - #!/usr/bin/env bash - set -eux -o pipefail - sed -i -e '/dhcp4: true$/!b' -e 'h;s/\S.*/dhcp-identifier: mac/;H;g' /etc/netplan/50-cloud-init.yaml - netplan apply -- path: /var/lib/cloud/scripts/per-instance/set_kernel_cmdline.sh - owner: root:root - permissions: '0777' - content: | - #!/usr/bin/env bash - set -eux -o pipefail - grub_file=${1:-"/etc/default/grub"} - kernel_parameters="intel_iommu=on iommu=pt" - sed -i~ "/^GRUB_CMDLINE_LINUX=/{h;s/\(=\".*\)\"/\1 ${kernel_parameters}\"/};\${x;/^$/{s//GRUB_CMDLINE_LINUX=\"${kernel_parameters}\"/;H};x}" "$grub_file" - update-grub - reboot -EOF -} - -function apply_userdata_credential { - name="$1" - cat < ./$name-user-data-credential.yaml -apiVersion: v1 -data: - userData: $(base64 -w 0 $name-userdata.yaml) -kind: Secret -metadata: - name: $name-user-data - namespace: metal3 -type: Opaque -EOF - kubectl apply -n metal3 -f $name-user-data-credential.yaml -} - -apply_networkdata_credential() { - name="$1" - cat < ./$name-network-data-credential.yaml -apiVersion: v1 -data: - networkData: $(base64 -w 0 $name-networkdata.json) -kind: Secret -metadata: - name: $name-network-data - namespace: metal3 -type: Opaque -EOF - kubectl apply -n metal3 -f $name-network-data-credential.yaml + node_networkdata $name >>${SCRIPTDIR}/machines-values.yaml } function make_bm_hosts { - kubectl create namespace metal3 --dry-run=client -o yaml | kubectl apply -f - while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do + printf " ${name}:\n" >>${SCRIPTDIR}/machines-values.yaml + printf " bmcUsername: ${ipmi_username}\n" >>${SCRIPTDIR}/machines-values.yaml + printf " bmcPassword: ${ipmi_password}\n" >>${SCRIPTDIR}/machines-values.yaml + printf " bmcAddress: ipmi://${ipmi_address}\n" >>${SCRIPTDIR}/machines-values.yaml + if [[ ! -z ${boot_mac} ]]; then + printf " bootMACAddress: ${boot_mac}\n" >>${SCRIPTDIR}/machines-values.yaml + fi + printf " imageName: ${BM_IMAGE}\n" >>${SCRIPTDIR}/machines-values.yaml create_userdata $name $os_username $os_password - apply_userdata_credential $name create_networkdata $name - apply_networkdata_credential $name - - GOPATH=$GOPATH:$(echo ${BMOPATH} | cut -d/ -f-2) GO111MODULE=auto \ - go run ${BMOPATH}/cmd/make-bm-worker/main.go \ - -address "ipmi://$ipmi_address" \ - -password "$ipmi_password" \ - -user "$ipmi_username" \ - -boot-mac "$boot_mac" \ - "$name" > $name-bm-node.yaml - - printf " image:" >> $name-bm-node.yaml - printf "\n url: ""%s" "$IMAGE_URL" >> $name-bm-node.yaml - printf "\n checksum: ""%s" "$IMAGE_CHECKSUM" >> $name-bm-node.yaml - printf "\n userData:" >> $name-bm-node.yaml - printf "\n name: ""%s" "$name""-user-data" >> $name-bm-node.yaml - printf "\n namespace: metal3" >> $name-bm-node.yaml - printf "\n networkData:" >> $name-bm-node.yaml - printf "\n name: ""%s" "$name""-network-data" >> $name-bm-node.yaml - printf "\n namespace: metal3" >> $name-bm-node.yaml - printf "\n rootDeviceHints:" >> $name-bm-node.yaml - printf "\n minSizeGigabytes: 48\n" >> $name-bm-node.yaml - kubectl apply -f $name-bm-node.yaml -n metal3 done } @@ -181,48 +84,32 @@ function configure_nodes { #make sure nodes.json file in /opt/ironic/ are configured if [ ! -f $IRONIC_DATA_DIR/nodes.json ]; then - cp $PWD/nodes.json.sample $IRONIC_DATA_DIR/nodes.json + cp ${SCRIPTDIR}/nodes.json.sample $IRONIC_DATA_DIR/nodes.json fi } -function remove_bm_hosts { +function deprovision_bm_hosts { while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do deprovision_compute_node $name done } -function cleanup { - while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do - kubectl delete --ignore-not-found=true bmh $name -n metal3 - kubectl delete --ignore-not-found=true secrets $name-bmc-secret -n metal3 - kubectl delete --ignore-not-found=true secrets $name-user-data -n metal3 - if [ -f $name-bm-node.yaml ]; then - rm -rf $name-bm-node.yaml - fi - - if [ -f $name-user-data-credential.yaml ]; then - rm -rf $name-user-data-credential.yaml - fi - - if [ -f $name-userdata.yaml ]; then - rm -rf $name-userdata.yaml - fi - done -} - function clean_all { - list_nodes | cleanup + helm -n metal3 uninstall machines + rm -f ${SCRIPTDIR}/machines-values.yaml if [ -f $IRONIC_DATA_DIR/nodes.json ]; then rm -rf $IRONIC_DATA_DIR/nodes.json fi } function apply_bm_hosts { + printf "machines:\n" >${SCRIPTDIR}/machines-values.yaml list_nodes | make_bm_hosts + helm -n metal3 install machines ${SCRIPTDIR}/../../machines --create-namespace -f ${SCRIPTDIR}/machines-values.yaml } function deprovision_all_hosts { - list_nodes | remove_bm_hosts + list_nodes | deprovision_bm_hosts } if [ "$1" == "deprovision" ]; then diff --git a/env/lib/common.sh b/env/lib/common.sh index 3f572c8..e4d8b08 100755 --- a/env/lib/common.sh +++ b/env/lib/common.sh @@ -98,6 +98,24 @@ function list_nodes { fi } +# Returns "null" when the field is not present +function networkdata_networks_field { + name=$1 + network=$2 + field=$3 + NODES_FILE="${IRONIC_DATA_DIR}/nodes.json" + cat $NODES_FILE | jq -c -r --arg name "$name" --arg network "$network" --arg field "$field" '.nodes[] | select(.name==$name) | .net.networks[] | select(.id==$network).'${field} +} + +# Returns "null" when the field is not present +function networkdata_links_field { + name=$1 + link=$2 + field=$3 + NODES_FILE="${IRONIC_DATA_DIR}/nodes.json" + cat $NODES_FILE | jq -c -r --arg name "$name" --arg link "$link" --arg field "$field" '.nodes[] | select(.name==$name) | .net.links[] | select(.id==$link).'${field} +} + function node_networkdata { name=$1 @@ -107,7 +125,30 @@ function node_networkdata { exit 1 fi - cat $NODES_FILE | jq -r --arg name "$name" '.nodes[] | select(.name==$name) | .net' + printf " networks:\n" + for network in $(cat $NODES_FILE | jq -r --arg name "$name" '.nodes[] | select(.name==$name) | .net.networks[].id'); do + link=$(networkdata_networks_field $name $network "link") + type=$(networkdata_networks_field $name $network "type") + mac=$(networkdata_links_field $name $link "ethernet_mac_address") + + # Optional values + ip_address=$(networkdata_networks_field $name $network "ip_address") + gateway=$(networkdata_networks_field $name $network "gateway") + dns_nameservers=$(networkdata_networks_field $name $network "dns_nameservers") + + printf " ${network}:\n" + printf " macAddress: ${mac}\n" + printf " type: ${type}\n" + if [[ $ip_address != "null" ]]; then + printf " ipAddress: ${ip_address}\n" + fi + if [[ $gateway != "null" ]]; then + printf " gateway: ${gateway}\n" + fi + if [[ $dns_nameservers != "null" ]]; then + printf " nameservers: ${dns_nameservers}\n" + fi + done } function wait_for { -- 2.16.6