From ec151f62b1977eb23da2c6b4ffd86b895e26e7ca Mon Sep 17 00:00:00 2001 From: "r.kuralamudhan" Date: Wed, 30 Oct 2019 06:15:21 +0000 Subject: [PATCH] update metal3 scripts Signed-off-by: r.kuralamudhan Change-Id: I98254ae96bad3192cf6f9bd8214ad4766b6f3109 --- deploy/metal3/scripts/01_metal3.sh | 211 ++++++++++++++++++++++++++++++++ deploy/metal3/scripts/02_verify.sh | 75 ++++++++++++ deploy/metal3/scripts/metal3.sh | 166 ------------------------- deploy/metal3/scripts/nodes.json.sample | 30 +++++ env/lib/common.sh | 32 +++-- env/metal3/01_install_package.sh | 5 +- env/metal3/02_configure.sh | 201 +++++++++++++++--------------- env/metal3/03_launch_prereq.sh | 50 +++++--- env/metal3/05_dhcp.conf.sample | 8 ++ env/metal3/06_host_cleanup.sh | 25 ++++ 10 files changed, 507 insertions(+), 296 deletions(-) create mode 100755 deploy/metal3/scripts/01_metal3.sh create mode 100755 deploy/metal3/scripts/02_verify.sh delete mode 100755 deploy/metal3/scripts/metal3.sh create mode 100644 deploy/metal3/scripts/nodes.json.sample create mode 100644 env/metal3/05_dhcp.conf.sample create mode 100755 env/metal3/06_host_cleanup.sh diff --git a/deploy/metal3/scripts/01_metal3.sh b/deploy/metal3/scripts/01_metal3.sh new file mode 100755 index 0000000..fa9e2ec --- /dev/null +++ b/deploy/metal3/scripts/01_metal3.sh @@ -0,0 +1,211 @@ +#!/bin/bash +set +ex + +LIBDIR="$(dirname "$(dirname "$(dirname "$PWD")")")" + +eval "$(go env)" + +source $LIBDIR/env/lib/common.sh + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +IMAGE_URL=http://172.22.0.1/images/${BM_IMAGE} +IMAGE_CHECKSUM=http://172.22.0.1/images/${BM_IMAGE}.md5sum + +function get_default_inteface_ipaddress { + local _ip=$1 + local _default_interface=$(awk '$2 == 00000000 { print $1 }' /proc/net/route) + local _ipv4address=$(ip addr show dev $_default_interface | awk '$1 == "inet" { sub("/.*", "", $2); print $2 }') + eval $_ip="'$_ipv4address'" +} + +function create_ssh_key { + #ssh key for compute node to communicate back to bootstrap server + mkdir -p $BUILD_DIR/ssh_key + ssh-keygen -C "compute.icn.akraino.lfedge.org" -f $BUILD_DIR/ssh_key/id_rsa + cat $BUILD_DIR/ssh_key/id_rsa.pub >> $HOME/.ssh/authorized_keys +} + +function set_compute_key { + _SSH_LOCAL_KEY=$(cat $BUILD_DIR/ssh_key/id_rsa) + cat << EOF +write_files: +- path: /opt/ssh_id_rsa + owner: root:root + permissions: '0600' + content: | + $_SSH_LOCAL_KEY +EOF +} + +function deprovision_compute_node { + name="$1" + kubectl patch baremetalhost $name -n metal3 --type merge \ + -p '{"spec":{"image":{"url":"","checksum":""}}}' +} + +function set_compute_ssh_config { + get_default_inteface_ipaddress default_addr + cat << EOF +- path: /root/.ssh/config + owner: root:root + permissions: '0600' + content: | + Host bootstrapmachine $default_addr + HostName $default_addr + IdentityFile /opt/ssh_id_rsa + User $USER +- path: /etc/apt/sources.list + owner: root:root + permissions: '0665' + content: | + deb [trusted=yes] ssh://$USER@$default_addr:$LOCAL_APT_REPO ./ +EOF +} + +function create_userdata { + name="$1" + COMPUTE_NODE_FQDN="$name.akraino.icn.org" + printf "#cloud-config\n" > $name-userdata.yaml + if [ -n "$COMPUTE_NODE_PASSWORD" ]; then + printf "password: ""%s" "$COMPUTE_NODE_PASSWORD" >> $name-userdata.yaml + printf "\nchpasswd: {expire: False}\n" >> $name-userdata.yaml + printf "ssh_pwauth: True\n" >> $name-userdata.yaml + fi + + if [ -n "$COMPUTE_NODE_FQDN" ]; then + printf "fqdn: ""%s" "$COMPUTE_NODE_FQDN" >> $name-userdata.yaml + printf "\n" >> $name-userdata.yaml + fi + printf "disable_root: false\n" >> $name-userdata.yaml + printf "ssh_authorized_keys:\n - " >> $name-userdata.yaml + + if [ ! -f $HOME/.ssh/id_rsa.pub ]; then + yes y | ssh-keygen -t rsa -N "" -f $HOME/.ssh/id_rsa + fi + + cat $HOME/.ssh/id_rsa.pub >> $name-userdata.yaml + network_config_files >> $name-userdata.yaml + printf "\n" >> $name-userdata.yaml +} + +function launch_baremetal_operator { + if [ ! -d $GOPATH/src/github.com/metal3-io/baremetal-operator ]; then + go get github.com/metal3-io/baremetal-operator + git checkout 3d40caa29dce82878d83aeb7f8dab4dc4a856160 + fi + + pushd $GOPATH/src/github.com/metal3-io/baremetal-operator + make deploy + popd +} + +function network_config_files { + cat << 'EOF' +write_files: +- path: /opt/ironic_net.sh + owner: root:root + permissions: '0777' + content: | + #!/usr/bin/env bash + set -xe + for intf in /sys/class/net/*; do + sudo ifconfig `basename $intf` up + sudo dhclient -nw `basename $intf` + done +runcmd: + - [ /opt/ironic_net.sh ] +EOF +} + +function apply_userdata_credential { + name="$1" + cat < ./$name-user-data-credential.yaml +apiVersion: v1 +data: + userData: $(base64 -w 0 $name-userdata.yaml) +kind: Secret +metadata: + name: $name-user-data + namespace: metal3 +type: Opaque +EOF + kubectl apply -n metal3 -f $name-user-data-credential.yaml +} + +function make_bm_hosts { + while read -r name username password address; do + create_userdata $name + apply_userdata_credential $name + + go run $GOPATH/src/github.com/metal3-io/baremetal-operator/cmd/make-bm-worker/main.go \ + -address "ipmi://$address" \ + -password "$password" \ + -user "$username" \ + "$name" > $name-bm-node.yaml + + printf " image:" >> $name-bm-node.yaml + printf "\n url: ""%s" "$IMAGE_URL" >> $name-bm-node.yaml + printf "\n checksum: ""%s" "$IMAGE_CHECKSUM" >> $name-bm-node.yaml + printf "\n userData:" >> $name-bm-node.yaml + printf "\n name: ""%s" "$name""-user-data" >> $name-bm-node.yaml + printf "\n namespace: metal3\n" >> $name-bm-node.yaml + kubectl apply -f $name-bm-node.yaml -n metal3 + done +} + +function configure_nodes { + if [ ! -d $IRONIC_DATA_DIR ]; then + mkdir -p $IRONIC_DATA_DIR + fi + + #make sure nodes.json file in /opt/ironic/ are configured + if [ ! -f $IRONIC_DATA_DIR/nodes.json ]; then + cp $PWD/nodes.json.sample $IRONIC_DATA_DIR/nodes.json + fi +} + +function remove_bm_hosts { + while read -r name username password address; do + deprovision_compute_node $name + done +} + +function apply_bm_hosts { + list_nodes | make_bm_hosts +} + +function deprovision_all_hosts { + list_nodes | remove_bm_hosts +} + +if [ "$1" == "launch" ]; then + launch_baremetal_operator + exit 0 +fi + +if [ "$1" == "deprovision" ]; then + deprovision_all_hosts + exit 0 +fi + +if [ "$1" == "provision" ]; then + apply_bm_hosts + exit 0 +fi + +echo "Usage: metal3.sh" +echo "launch - Launch the metal3 operator" +echo "provision - provision baremetal node as specified in common.sh" +echo "deprovision - deprovision baremetal node as specified in common.sh" +exit 1 + +#Following code is tested for the offline mode +#Will be intergrated for the offline mode for ICNi v.0.1.0 beta +#create_ssh_key +#create_userdata +#set_compute_key +#set_compute_ssh_config diff --git a/deploy/metal3/scripts/02_verify.sh b/deploy/metal3/scripts/02_verify.sh new file mode 100755 index 0000000..fff486b --- /dev/null +++ b/deploy/metal3/scripts/02_verify.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +#set -x + +LIBDIR="$(dirname "$(dirname "$(dirname "$PWD")")")" + +eval "$(go env)" + +source $LIBDIR/env/lib/common.sh + +declare -i timeout=60 +declare -i interval=60 + +function check_provisioned { + declare -i prev_host_state=0 + declare -i j=0 + echo "Baremetal state: 1 means provisioned & 0 means not yet provisioned" + while read -r name username password address; do + declare -i current_host_state=0 + state=$(kubectl get baremetalhosts $name -n metal3 -o json | jq -r '.status.provisioning.state') + echo "Baremetal host metal3 state - "$name" : "$state + + if [ $state == "provisioned" ];then + current_host_state=1 + fi + + echo "Baremetal $name current_host_state : "$current_host_state + echo "Previous Baremetals prev_host_state : "$prev_host_state + + if [ $j -eq 0 ]; then + prev_host_state=$current_host_state + ((j+=1)) + continue + fi + + if [ $current_host_state -eq 1 ] && [ $prev_host_state -eq 1 ]; then + prev_host_state=1 + else + prev_host_state=0 + fi + + echo "All Baremetal hosts aggregated state - prev_host_state:"$prev_host_state + ((j+=1)) + done + return $prev_host_state +} + +function warm_up_time { + echo "Wait for 240s for all baremetal hosts to reboot and network is up" + sleep 4m +} + +function wait_for_provisioned { + all_bmh_provisioned=1 + declare -i k=1 + while ((timeout > 0)); do + echo "Try $k iteration : Wait for $interval seconds to check all bmh state" + sleep $interval + list_nodes | check_provisioned + all_bmh_state=$? + if [[ $all_bmh_state -eq $all_bmh_provisioned ]]; then + echo "All the Baremetal hosts are provisioned - success" + warm_up_time + exit 0 + fi + ((timeout-=1)) + ((k+=1)) + done + exit 1 +} + +function verify_bm_hosts { + wait_for_provisioned +} + +verify_bm_hosts diff --git a/deploy/metal3/scripts/metal3.sh b/deploy/metal3/scripts/metal3.sh deleted file mode 100755 index 48f350e..0000000 --- a/deploy/metal3/scripts/metal3.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash - -LIBDIR="$(dirname "$(dirname "$(dirname "$PWD")")")" - -eval "$(go env)" - -BM_OPERATOR="${BM_OPERATOR:-https://github.com/metal3-io/baremetal-operator.git}" - -source $LIBDIR/env/lib/common.sh - -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" - exit 1 -fi - -function get_default_inteface_ipaddress { - local _ip=$1 - local _default_interface=$(awk '$2 == 00000000 { print $1 }' /proc/net/route) - local _ipv4address=$(ip addr show dev $_default_interface | awk '$1 == "inet" { sub("/.*", "", $2); print $2 }') - eval $_ip="'$_ipv4address'" -} - -create_ssh_key() { - #ssh key for compute node to communicate back to bootstrap server - mkdir -p $BUILD_DIR/ssh_key - ssh-keygen -C "compute.icn.akraino.lfedge.org" -f $BUILD_DIR/ssh_key/id_rsa - cat $BUILD_DIR/ssh_key/id_rsa.pub >> $HOME/.ssh/authorized_keys -} - -set_compute_key() { -_SSH_LOCAL_KEY=$(cat $BUILD_DIR/ssh_key/id_rsa) -cat << EOF -write_files: -- path: /opt/ssh_id_rsa - owner: root:root - permissions: '0600' - content: | - $_SSH_LOCAL_KEY -EOF -} - -provision_compute_node() { - IMAGE_URL=http://172.22.0.1/images/${BM_IMAGE} - IMAGE_CHECKSUM=http://172.22.0.1/images/${BM_IMAGE}.md5sum - - if [ ! -d $GOPATH/src/github.com/metal3-io/baremetal-operator ]; then - go get github.com/metal3-io/baremetal-operator - fi - - go run $GOPATH/src/github.com/metal3-io/baremetal-operator/cmd/make-bm-worker/main.go \ - -address "ipmi://$COMPUTE_IPMI_ADDRESS" \ - -user "$COMPUTE_IPMI_USER" \ - -password "$COMPUTE_IPMI_PASSWORD" \ - "$COMPUTE_NODE_NAME" > $COMPUTE_NODE_NAME-bm-node.yaml - - printf " image:" >> $COMPUTE_NODE_NAME-bm-node.yaml - printf "\n url: ""%s" "$IMAGE_URL" >> $COMPUTE_NODE_NAME-bm-node.yaml - printf "\n checksum: ""%s" "$IMAGE_CHECKSUM" >> $COMPUTE_NODE_NAME-bm-node.yaml - printf "\n userData:" >> $COMPUTE_NODE_NAME-bm-node.yaml - printf "\n name: ""%s" "$COMPUTE_NODE_NAME""-user-data" >> $COMPUTE_NODE_NAME-bm-node.yaml - printf "\n namespace: metal3\n" >> $COMPUTE_NODE_NAME-bm-node.yaml - kubectl apply -f $COMPUTE_NODE_NAME-bm-node.yaml -n metal3 -} - -deprovision_compute_node() { - kubectl patch baremetalhost $COMPUTE_NODE_NAME -n metal3 --type merge \ - -p '{"spec":{"image":{"url":"","checksum":""}}}' -} - -set_compute_ssh_config() { -get_default_inteface_ipaddress default_addr -cat << EOF -- path: /root/.ssh/config - owner: root:root - permissions: '0600' - content: | - Host bootstrapmachine $default_addr - HostName $default_addr - IdentityFile /opt/ssh_id_rsa - User $USER -- path: /etc/apt/sources.list - owner: root:root - permissions: '0665' - content: | - deb [trusted=yes] ssh://$USER@$default_addr:$LOCAL_APT_REPO ./ -EOF -} - -create_userdata() { - printf "#cloud-config\n" > userdata.yaml - if [ -n "$COMPUTE_NODE_PASSWORD" ]; then - printf "password: ""%s" "$COMPUTE_NODE_PASSWORD" >> userdata.yaml - printf "\nchpasswd: {expire: False}\n" >> userdata.yaml - printf "ssh_pwauth: True\n" >> userdata.yaml - fi - - if [ -n "$COMPUTE_NODE_FQDN" ]; then - printf "fqdn: ""%s" "$COMPUTE_NODE_FQDN" >> userdata.yaml - printf "\n" >> userdata.yaml - fi - printf "disable_root: false\n" >> userdata.yaml - printf "ssh_authorized_keys:\n - " >> userdata.yaml - - if [ ! -f $HOME/.ssh/id_rsa.pub ]; then - yes y | ssh-keygen -t rsa -N "" -f $HOME/.ssh/id_rsa - fi - - cat $HOME/.ssh/id_rsa.pub >> userdata.yaml - printf "\n" >> userdata.yaml -} - -apply_userdata_credential() { - cat < ./$COMPUTE_NODE_NAME-user-data.yaml -apiVersion: v1 -data: - userData: $(base64 -w 0 userdata.yaml) -kind: Secret -metadata: - name: $COMPUTE_NODE_NAME-user-data - namespace: metal3 -type: Opaque -EOF - kubectl apply -n metal3 -f $COMPUTE_NODE_NAME-user-data.yaml -} - -launch_baremetal_operator() { - if [ ! -d $GOPATH/src/github.com/metal3-io/baremetal-operator ]; then - go get github.com/metal3-io/baremetal-operator - fi - - pushd $GOPATH/src/github.com/metal3-io/baremetal-operator - make deploy - popd - -} - -if [ "$1" == "launch" ]; then - launch_baremetal_operator - exit 0 -fi - -if [ "$1" == "deprovision" ]; then - deprovision_compute_node - exit 0 -fi - -if [ "$1" == "provision" ]; then - create_userdata - apply_userdata_credential - provision_compute_node - exit 0 -fi - - -echo "Usage: metal3.sh" -echo "launch - Launch the metal3 operator" -echo "provision - provision baremetal node as specified in common.sh" -echo "deprovision - deprovision baremetal node as specified in common.sh" -exit 1 - -#Following code is tested for the offline mode -#Will be intergrated for the offline mode for ICNi v.0.1.0 beta -#create_ssh_key -#create_userdata -#set_compute_key -#set_compute_ssh_config diff --git a/deploy/metal3/scripts/nodes.json.sample b/deploy/metal3/scripts/nodes.json.sample new file mode 100644 index 0000000..d1c81c3 --- /dev/null +++ b/deploy/metal3/scripts/nodes.json.sample @@ -0,0 +1,30 @@ +{ + "nodes": [ + { + "name": "edge01-node01", + "ipmi_driver_info": { + "username": "admin", + "password": "admin", + "address": "10.10.10.11" + }, + "os": { + "image_name": "bionic-server-cloudimg-amd64.img", + "username": "ubuntu", + "password": "mypasswd" + } + }, + { + "name": "edge01-node02", + "ipmi_driver_info": { + "username": "admin", + "password": "admin", + "address": "10.10.10.12" + }, + "os": { + "image_name": "bionic-server-cloudimg-amd64.img", + "username": "ubuntu", + "password": "mypasswd" + } + } + ] +} diff --git a/env/lib/common.sh b/env/lib/common.sh index 5705402..e3a8bd6 100755 --- a/env/lib/common.sh +++ b/env/lib/common.sh @@ -17,8 +17,9 @@ POD_NETWORK_CIDR=${POD_NETWORK_CIDR:-"10.244.0.0/16"} PODMAN_CNI_CONFLIST=${PODMAN_CNI_CONFLIST:-"https://raw.githubusercontent.com/containers/libpod/v1.4.4/cni/87-podman-bridge.conflist"} #Bootstrap K8s cluster -BS_DHCP_INTERFACE=${BS_DHCP_INTERFACE:-"eno2"} +BS_DHCP_INTERFACE=${BS_DHCP_INTERFACE:-"ens513f0"} BS_DHCP_INTERFACE_IP=${BS_DHCP_INTERFACE_IP:-"172.31.1.1/24"} +BS_DHCP_DIR=${BS_DHCP_DIR:-$DOWNLOAD_PATH/dhcp} #Ironic variables IRONIC_IMAGE=${IRONIC_IMAGE:-"quay.io/metal3-io/ironic:master"} @@ -28,20 +29,16 @@ IRONIC_BAREMETAL_SOCAT_IMAGE=${IRONIC_BAREMETAL_SOCAT_IMAGE:-"alpine/socat:lates IRONIC_DATA_DIR=${IRONIC_DATA_DIR:-"/opt/ironic"} #IRONIC_PROVISIONING_INTERFACE is required to be provisioning, don't change it +IRONIC_INTERFACE=${IRONIC_INTERFACE:-"enp4s0f1"} IRONIC_PROVISIONING_INTERFACE=${IRONIC_PROVISIONING_INTERFACE:-"provisioning"} -IRONIC_IPMI_INTERFACE=${IRONIC_IPMI_INTERFACE:-"eno1"} +IRONIC_IPMI_INTERFACE=${IRONIC_IPMI_INTERFACE:-"enp4s0f0"} IRONIC_PROVISIONING_INTERFACE_IP=${IRONIC_PROVISIONING_INTERFACE_IP:-"172.22.0.1"} -IRONIC_IPMI_INTERFACE_IP=${IRONIC_IPMI_INTERFACE_IP:-"172.31.1.9"} +IRONIC_IPMI_INTERFACE_IP=${IRONIC_IPMI_INTERFACE_IP:-"10.10.110.20"} BM_IMAGE_URL=${BM_IMAGE_URL:-"https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img"} BM_IMAGE=${BM_IMAGE:-"bionic-server-cloudimg-amd64.img"} #Todo change into nodes list in json pattern -COMPUTE_NODE_NAME=${COMPUTE_NODE_NAME:-"el-100-node-01"} -COMPUTE_IPMI_ADDRESS=${COMPUTE_IPMI_ADDRESS:-"172.31.1.17"} -COMPUTE_IPMI_USER=${COMPUTE_IPMI_USER:-"ryeleswa"} -COMPUTE_IPMI_PASSWORD=${COMPUTE_IPMI_PASSWORD:-"changeme1"} -COMPUTE_NODE_FQDN=${COMPUTE_NODE_FQDN:-"node01.akraino.org"} -#COMPUTE_NODE_HOSTNAME=${COMPUTE_NODE_HOSTNAME:-"node01"} +COMPUTE_NODE_FQDN=${COMPUTE_NODE_FQDN:-".akraino.org"} COMPUTE_NODE_PASSWORD=${COMPUTE_NODE_PASSWORD:-"mypasswd"} #refered from onap @@ -74,3 +71,20 @@ function call_api { fi fi } + +function list_nodes() { + NODES_FILE="${IRONIC_DATA_DIR}/nodes.json" + cat "$NODES_FILE" | \ + jq '.nodes[] | { + name, + username:.ipmi_driver_info.username, + password:.ipmi_driver_info.password, + address:.ipmi_driver_info.address + } | + .name + " " + + .username + " " + + .password + " " + + .address' \ + | sed 's/"//g' +} + diff --git a/env/metal3/01_install_package.sh b/env/metal3/01_install_package.sh index 008bd2b..8aa458f 100755 --- a/env/metal3/01_install_package.sh +++ b/env/metal3/01_install_package.sh @@ -27,9 +27,10 @@ function install_essential_packages { vim \ wget \ git \ - software-properties-common + software-properties-common \ + bridge-utils - add-apt-repository ppa:longsleep/golang-backports + add-apt-repository -y ppa:longsleep/golang-backports apt-get update apt-get install golang-go } diff --git a/env/metal3/02_configure.sh b/env/metal3/02_configure.sh index 97c89d8..eabf780 100755 --- a/env/metal3/02_configure.sh +++ b/env/metal3/02_configure.sh @@ -10,9 +10,9 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi -function check_inteface_ip() { - local interface=$1 - local ipaddr=$2 +function check_inteface_ip { + local interface=$1 + local ipaddr=$2 if [ ! $(ip addr show dev $interface) ]; then exit 1 @@ -24,138 +24,137 @@ function check_inteface_ip() { fi } -function configure_dhcp_bridge() { - brctl addbr dhcp0 - ip link set dhcp0 up - brctl addif dhcp0 $BS_DHCP_INTERFACE - ip addr add dev dhcp0 $BS_DHCP_INTERFACE_IP +function configure_dhcp_bridge { + brctl addbr dhcp0 + ip link set dhcp0 up + brctl addif dhcp0 $BS_DHCP_INTERFACE + ip addr add dev dhcp0 $BS_DHCP_INTERFACE_IP } -function configure_ironic_bridge() { - brctl addbr provisioning - ip link set provisioning up - brctl addif provisioning $IRONIC_IPMI_INTERFACE - ip addr add dev provisioning 172.22.0.1/24 +function configure_ironic_bridge { + brctl addbr provisioning + ip link set provisioning up + brctl addif provisioning $IRONIC_INTERFACE + ip addr add dev provisioning 172.22.0.1/24 } -function configure_kubelet() { - swapoff -a - #Todo addition kubelet configuration +function configure_kubelet { + swapoff -a + #Todo addition kubelet configuration } -function configure_kubeadm() { - #Todo error handing - if [ "$1" == "offline" ]; then - for images in kube-apiserver kube-controller-manager kube-scheduler kube-proxy; do - docker load --input $CONTAINER_IMAGES_DIR/$images.tar; - done +function configure_kubeadm { + #Todo error handing + if [ "$1" == "offline" ]; then + for images in kube-apiserver kube-controller-manager kube-scheduler kube-proxy; do + docker load --input $CONTAINER_IMAGES_DIR/$images.tar; + done - docker load --input $CONTAINER_IMAGES_DIR/pause.tar - docker load --input $CONTAINER_IMAGES_DIR/etcd.tar - docker load --input $CONTAINER_IMAGES_DIR/coredns.tar + docker load --input $CONTAINER_IMAGES_DIR/pause.tar + docker load --input $CONTAINER_IMAGES_DIR/etcd.tar + docker load --input $CONTAINER_IMAGES_DIR/coredns.tar return fi - kubeadm config images pull --kubernetes-version=$KUBE_VERSION + kubeadm config images pull --kubernetes-version=$KUBE_VERSION } -function configure_ironic_interfaces() { - #Todo later to change the CNI networking for podman networking - # Add firewall rules to ensure the IPA ramdisk can reach httpd, Ironic and the Inspector API on the host - if [ "$IRONIC_PROVISIONING_INTERFACE" ]; then - check_inteface_ip $IRONIC_PROVISIONING_INTERFACE $IRONIC_PROVISIONING_INTERFACE_IP - else - exit 1 - - fi +function configure_ironic_interfaces { + #Todo later to change the CNI networking for podman networking + # Add firewall rules to ensure the IPA ramdisk can reach httpd, Ironic and the Inspector API on the host + if [ "$IRONIC_PROVISIONING_INTERFACE" ]; then + check_inteface_ip $IRONIC_PROVISIONING_INTERFACE $IRONIC_PROVISIONING_INTERFACE_IP + else + exit 1 + fi - if [ "$IRONIC_IPMI_INTERFACE" ]; then + if [ "$IRONIC_IPMI_INTERFACE" ]; then check_inteface_ip $IRONIC_IPMI_INTERFACE $IRONIC_IPMI_INTERFACE_IP else exit 1 fi - for port in 80 5050 6385 ; do - if ! sudo iptables -C INPUT -i $IRONIC_PROVISIONING_INTERFACE -p tcp -m tcp --dport $port -j ACCEPT > /dev/null 2>&1; then - sudo iptables -I INPUT -i $IRONIC_PROVISIONING_INTERFACE -p tcp -m tcp --dport $port -j ACCEPT - fi - done + for port in 80 5050 6385 ; do + if ! sudo iptables -C INPUT -i $IRONIC_PROVISIONING_INTERFACE -p tcp -m tcp --dport $port -j ACCEPT > /dev/null 2>&1; then + sudo iptables -I INPUT -i $IRONIC_PROVISIONING_INTERFACE -p tcp -m tcp --dport $port -j ACCEPT + fi + done - # Allow ipmi to the bmc processes - if ! sudo iptables -C INPUT -i $IRONIC_IPMI_INTERFACE -p udp -m udp --dport 6230:6235 -j ACCEPT 2>/dev/null ; then - sudo iptables -I INPUT -i $IRONIC_IPMI_INTERFACE -p udp -m udp --dport 6230:6235 -j ACCEPT - fi + # Allow ipmi to the bmc processes + if ! sudo iptables -C INPUT -i $IRONIC_IPMI_INTERFACE -p udp -m udp --dport 6230:6235 -j ACCEPT 2>/dev/null ; then + sudo iptables -I INPUT -i $IRONIC_IPMI_INTERFACE -p udp -m udp --dport 6230:6235 -j ACCEPT + fi - #Allow access to dhcp and tftp server for pxeboot - for port in 67 69 ; do - if ! sudo iptables -C INPUT -i $IRONIC_PROVISIONING_INTERFACE -p udp --dport $port -j ACCEPT 2>/dev/null ; then - sudo iptables -I INPUT -i $IRONIC_PROVISIONING_INTERFACE -p udp --dport $port -j ACCEPT - fi - done + #Allow access to dhcp and tftp server for pxeboot + for port in 67 69 ; do + if ! sudo iptables -C INPUT -i $IRONIC_PROVISIONING_INTERFACE -p udp --dport $port -j ACCEPT 2>/dev/null ; then + sudo iptables -I INPUT -i $IRONIC_PROVISIONING_INTERFACE -p udp --dport $port -j ACCEPT + fi + done } -function configure_ironic_offline() { - if [ ! -d $CONTAINER_IMAGES_DIR ] && [ ! -d $BUILD_DIR ]; then - exit 1 - fi - - for image in ironic-inspector-image ironic-image podman-pause \ - baremetal-operator socat; do - if [ ! -f "$CONTAINER_IMAGES_DIR/$image" ]; then - exit 1 - fi - done +function configure_ironic_offline { + if [ ! -d $CONTAINER_IMAGES_DIR ] && [ ! -d $BUILD_DIR ]; then + exit 1 + fi - if [ ! -f "$BUILD_DIR/ironic-python-agent.initramfs"] && [ ! -f \ - "$BUILD_DIR/ironic-python-agent.kernel" ] && [ ! -f - "$BUILD_DIR/$BM_IMAGE" ]; then - exit 1 + for image in ironic-inspector-image ironic-image podman-pause \ + baremetal-operator socat; do + if [ ! -f "$CONTAINER_IMAGES_DIR/$image" ]; then + exit 1 fi + done + + if [ ! -f "$BUILD_DIR/ironic-python-agent.initramfs"] && [ ! -f \ + "$BUILD_DIR/ironic-python-agent.kernel" ] && [ ! -f + "$BUILD_DIR/$BM_IMAGE" ]; then + exit 1 + fi - podman load --input $CONTAINER_IMAGES_DIR/ironic-inspector-image.tar - podman load --input $CONTAINER_IMAGES_DIR/ironic-image.tar - podman load --input $CONTAINER_IMAGES_DIR/podman-pause.tar + podman load --input $CONTAINER_IMAGES_DIR/ironic-inspector-image.tar + podman load --input $CONTAINER_IMAGES_DIR/ironic-image.tar + podman load --input $CONTAINER_IMAGES_DIR/podman-pause.tar - docker load --input $CONTAINER_IMAGES_DIR/baremetal-operator.tar - docker load --input $CONTAINER_IMAGES_DIR/socat.tar + docker load --input $CONTAINER_IMAGES_DIR/baremetal-operator.tar + docker load --input $CONTAINER_IMAGES_DIR/socat.tar - mkdir -p "$IRONIC_DATA_DIR/html/images" + mkdir -p "$IRONIC_DATA_DIR/html/images" - cp $BUILD_DIR/ironic-python-agent.initramfs $IRONIC_DATA_DIR/html/images/ - cp $BUILD_DIR/ironic-python-agent.kernel $IRONIC_DATA_DIR/html/images/ - cp $BUILD_DIR/$BM_IMAGE $IRONIC_DATA_DIR/html/images/ - md5sum $BUILD_DIR/$BM_IMAGE | awk '{print $1}' > $BUILD_DIR/${BM_IMAGE}.md5sum + cp $BUILD_DIR/ironic-python-agent.initramfs $IRONIC_DATA_DIR/html/images/ + cp $BUILD_DIR/ironic-python-agent.kernel $IRONIC_DATA_DIR/html/images/ + cp $BUILD_DIR/$BM_IMAGE $IRONIC_DATA_DIR/html/images/ + md5sum $BUILD_DIR/$BM_IMAGE | awk '{print $1}' > $BUILD_DIR/${BM_IMAGE}.md5sum } -function configure_ironic() { - if [ "$1" == "offline" ]; then - configure_ironic_offline - return - fi +function configure_ironic { + if [ "$1" == "offline" ]; then + configure_ironic_offline + return + fi - podman pull $IRONIC_IMAGE - podman pull $IRONIC_INSPECTOR_IMAGE - - mkdir -p "$IRONIC_DATA_DIR/html/images" - pushd $IRONIC_DATA_DIR/html/images - - if [ ! -f ironic-python-agent.initramfs ]; then - curl --insecure --compressed -L https://images.rdoproject.org/master/rdo_trunk/current-tripleo-rdo/ironic-python-agent.tar | tar -xf - - fi - - if [[ "$BM_IMAGE_URL" && "$BM_IMAGE" ]]; then + podman pull $IRONIC_IMAGE + podman pull $IRONIC_INSPECTOR_IMAGE + + mkdir -p "$IRONIC_DATA_DIR/html/images" + pushd $IRONIC_DATA_DIR/html/images + + if [ ! -f ironic-python-agent.initramfs ]; then + curl --insecure --compressed -L https://images.rdoproject.org/master/rdo_trunk/current-tripleo-rdo/ironic-python-agent.tar | tar -xf - + fi + + if [[ "$BM_IMAGE_URL" && "$BM_IMAGE" ]]; then curl -o ${BM_IMAGE} --insecure --compressed -O -L ${BM_IMAGE_URL} md5sum ${BM_IMAGE} | awk '{print $1}' > ${BM_IMAGE}.md5sum - fi - popd + fi + popd } -function configure() { - configure_kubeadm $1 - configure_kubelet - configure_ironic_interfaces - configure_ironic $1 - configure_dhcp_bridge - configure_ironic_bridge +function configure { + configure_kubeadm $1 + configure_kubelet + configure_ironic $1 + configure_dhcp_bridge + configure_ironic_bridge + configure_ironic_interfaces } if [ "$1" == "-o" ]; then diff --git a/env/metal3/03_launch_prereq.sh b/env/metal3/03_launch_prereq.sh index e48b7ee..771c765 100755 --- a/env/metal3/03_launch_prereq.sh +++ b/env/metal3/03_launch_prereq.sh @@ -18,34 +18,34 @@ function get_default_inteface_ipaddress { eval $_ip="'$_ipv4address'" } - - function check_cni_network { #since bootstrap cluster is a single node cluster, #podman and bootstap cluster have same network configuration to avoid the cni network conf conflicts if [ ! -d "/etc/cni/net.d" ]; then - mkdir -p "/etc/cni/net.d" + mkdir -p "/etc/cni/net.d" + fi + + if [ -f "/etc/cni/net.d/87-podman-bridge.conflist" ]; then + rm -rf /etc/cni/net.d/87-podman-bridge.conflist fi - if [ ! -f "/etc/cni/net.d/87-podman-bridge.conflist" ]; then if [ "$1" == "offline" ]; then cp $BUILD_DIR/87-podman-bridge.conflist /etc/cni/net.d/ return - fi + fi if !(wget $PODMAN_CNI_CONFLIST -P /etc/cni/net.d/); then exit 1 fi - fi } function create_k8s_regular_user { if [ ! -d "$HOME/.kube" ]; then - mkdir -p $HOME/.kube + mkdir -p $HOME/.kube fi if [ ! -f /etc/kubernetes/admin.conf]; then - exit 1 + exit 1 fi cp -rf /etc/kubernetes/admin.conf $HOME/.kube/config @@ -60,19 +60,19 @@ function check_k8s_node_status { check_node=$(kubectl get node -o \ jsonpath='{.items[0].status.conditions[?(@.reason == "KubeletReady")].status}') if [ $check_node != "" ]; then - node_status=${check_node} + node_status=${check_node} fi if [ $node_status == "True" ]; then - break + break fi sleep 3 done if [ $node_status != "True" ]; then - echo "bootstrap cluster single node status is not ready" - exit 1 + echo "bootstrap cluster single node status is not ready" + exit 1 fi } @@ -85,7 +85,7 @@ function install_podman { # Start dnsmasq, http, mariadb, and ironic containers using same image podman run -d --net host --privileged --name dnsmasq --pod ironic-pod \ - -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/rundnsmasq ${IRONIC_IMAGE} + -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/rundnsmasq ${IRONIC_IMAGE} podman run -d --net host --privileged --name httpd --pod ironic-pod \ -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/runhttpd ${IRONIC_IMAGE} @@ -100,25 +100,38 @@ function install_podman { # Start Ironic Inspector podman run -d --net host --privileged --name ironic-inspector \ - --pod ironic-pod "${IRONIC_INSPECTOR_IMAGE}" + --pod ironic-pod "${IRONIC_INSPECTOR_IMAGE}" } function remove_k8s_noschedule_taint { #Bootstrap cluster is a single node nodename=$(kubectl get node -o jsonpath='{.items[0].metadata.name}') if !(kubectl taint node $nodename node-role.kubernetes.io/master:NoSchedule-); then - exit 1 + exit 1 fi } function install_k8s_single_node { get_default_inteface_ipaddress apiserver_advertise_addr kubeadm_init="kubeadm init --kubernetes-version=$KUBE_VERSION \ - --pod-network-cidr=$POD_NETWORK_CIDR \ - --apiserver-advertise-address=$apiserver_advertise_addr" + --pod-network-cidr=$POD_NETWORK_CIDR \ + --apiserver-advertise-address=$apiserver_advertise_addr" if !(${kubeadm_init}); then - exit 1 + exit 1 + fi +} + +function install_dhcp { + if [ ! -d $BS_DHCP_DIR ]; then + mkdir -p $BS_DHCP_DIR fi + + #make sure the dhcp conf sample are configured + if [ ! -f $BS_DHCP_DIR/dhcpd.conf ]; then + cp $PWD/05_dhcp.conf.sample $BS_DHCP_DIR/dhcpd.conf + fi + + kubectl create -f $PWD/04_dhcp.yaml } function install { @@ -132,6 +145,7 @@ function install { #install_podman #Todo - error handling mechanism install_podman + install_dhcp } if [ "$1" == "-o" ]; then diff --git a/env/metal3/05_dhcp.conf.sample b/env/metal3/05_dhcp.conf.sample new file mode 100644 index 0000000..7aaa3e6 --- /dev/null +++ b/env/metal3/05_dhcp.conf.sample @@ -0,0 +1,8 @@ +default-lease-time 3600; +max-lease-time 7200; +authoritative; +subnet 172.31.1.0 netmask 255.255.255.0 { + option routers 172.31.1.1; + option subnet-mask 255.255.255.0; + range 172.31.1.201 172.31.1.254; +} diff --git a/env/metal3/06_host_cleanup.sh b/env/metal3/06_host_cleanup.sh new file mode 100755 index 0000000..03fca87 --- /dev/null +++ b/env/metal3/06_host_cleanup.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -x +LIBDIR="$(dirname "$PWD")" + +source $LIBDIR/lib/common.sh + +# Kill and remove the running ironic containers +for name in ironic ironic-inspector dnsmasq httpd mariadb; do + sudo podman ps | grep -w "$name$" && sudo podman kill $name + sudo podman ps --all | grep -w "$name$" && sudo podman rm $name -f +done + +# Remove existing pod +if sudo podman pod exists ironic-pod ; then + sudo podman pod rm ironic-pod -f +fi + +ip link set provisioning down +brctl delbr provisioning + +ip link set dhcp0 down +brctl delbr dhcp0 + +rm -rf ${BS_DHCP_DIR} +rm -rf ${IRONIC_DATA_DIR} -- 2.16.6