From: Todd Malsbary Date: Mon, 26 Apr 2021 20:20:43 +0000 (-0700) Subject: Add explicit network configuration to nodes.json X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=31a860f56b53eb51a7dc4189e6959a004733115e;p=icn.git Add explicit network configuration to nodes.json A "net" section is added to nodes.json to provide the value of the networkData field to the created BareMetalHost resource. This allows static configuration of a given host's interface, for example. Due to limitations with the networkData field, all interfaces must now be explicitly listed. Previously, dhclient was run against all interfaces on the host and gateway and DNS information was provided via the PROVIDER_NETWORK variables in user_config.sh. Now that information must be provided in nodes.json. Issue-ID: ICN-575 Signed-off-by: Todd Malsbary Change-Id: Icbf29856a0e1209bd1bcfe7da563f8e6cc1057cd --- diff --git a/README.md b/README.md index d96ae78..b833ff3 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,11 @@ command `make install`. The user is required to provide the IPMI information of the servers they connect to the Local Controller by editing node JSON sample file in the directory icn/deploy/metal3/scripts/nodes.json.sample as -below. This example only shows 2 servers. If you want to increase -servers, just add another array. +below. This example only shows 2 servers, statically configured on the +baremetal network. If you want to increase servers, just add another +array. If the baremetal network provides a DHCP server with gateway +and DNS server information, just change the baremetal type to "ipv4". +ICN provides DHCP servers for the provisioning and bootstrap networks. `node.json.sample` ``` json @@ -149,6 +152,46 @@ servers, just add another array. "image_name": "bionic-server-cloudimg-amd64.img", "username": "ubuntu", "password": "mypasswd" + }, + "net": { + "links": [ + { + "id": "baremetal_nic", + "ethernet_mac_address": "00:1e:67:fe:f4:19", + "type": "phy" + }, + { + "id": "provisioning_nic", + "ethernet_mac_address": "00:1e:67:fe:f4:1a", + "type": "phy" + }, + { + "id": "bootstrap_nic", + "ethernet_mac_address": "00:1e:67:f8:6a:40", + "type": "phy" + } + ], + "networks": [ + { + "id": "baremetal", + "link": "baremetal_nic", + "type": "ipv4", + "ip_address": "10.10.110.21/24", + "gateway": "10.10.110.1", + "dns_nameservers": ["8.8.8.8"] + }, + { + "id": "provisioning", + "link": "provisioning_nic", + "type": "ipv4_dhcp" + }, + { + "id": "bootstrap", + "link": "bootstrap_nic", + "type": "ipv4_dhcp" + } + ], + "services": [] } }, { @@ -162,6 +205,46 @@ servers, just add another array. "image_name": "bionic-server-cloudimg-amd64.img", "username": "ubuntu", "password": "mypasswd" + }, + "net": { + "links": [ + { + "id": "baremetal_nic", + "ethernet_mac_address": "00:1e:67:f1:5b:90", + "type": "phy" + }, + { + "id": "bootstrap_nic", + "ethernet_mac_address": "00:1e:67:f8:69:80", + "type": "phy" + }, + { + "id": "provisioning_nic", + "ethernet_mac_address": "00:1e:67:f1:5b:91", + "type": "phy" + } + ], + "networks": [ + { + "id": "baremetal", + "link": "baremetal_nic", + "type": "ipv4", + "ip_address": "10.10.110.22/24", + "gateway": "10.10.110.1", + "dns_nameservers": ["8.8.8.8"] + }, + { + "id": "provisioning", + "link": "provisioning_nic", + "type": "ipv4_dhcp" + }, + { + "id": "bootstrap", + "link": "bootstrap_nic", + "type": "ipv4_dhcp" + } + ], + "services": [] } }] } @@ -183,6 +266,27 @@ servers, just add another array. - *image_name*: Images name should be in qcow2 format. - *username*: Login username for the OS provisioned. - *password*: Login password for the OS provisioned. +- *net*: Bare metal network information is a json field. It describes + the interfaces and networks used by ICN. For more information, + refer to the *networkData* field of the BareMetalHost resource + definition. + - *links*: An array of interfaces. + - *id*: The ID of the interface. This is used in the network + definitions to associate the interface with its network + configuration. + - *ethernet_mac_address*: The MAC address of the interface. + - *type*: The type of interface. Valid values are "phy". + - *networks*: An array of networks. + - *id*: The ID of the network. + - *link*: The ID of the link this network definition applies to. + - *type*: The type of network, either dynamic ("ipv4_dhcp") or + static ("ipv4"). + - *ip_address*: Only valid for type "ipv4"; the IP address of the + interface. + - *gateway*: Only valid for type "ipv4"; the gateway of this + network. + - *dns_nameservers*: Only valid for type "ipv4"; an array of DNS + servers. #### Creating the Settings Files @@ -203,9 +307,8 @@ export BS_DHCP_INTERFACE_IP="172.31.1.1/24" #Edge Location Provider Network configuration #Net A - Provider Network -#If provider having specific Gateway and DNS server details in the edge location -#export PROVIDER_NETWORK_GATEWAY="10.10.110.1" -#export PROVIDER_NETWORK_DNS="8.8.8.8" +#If provider having specific Gateway and DNS server details in the edge location, +#supply those values in nodes.json. #Ironic Metal3 settings for provisioning network #Interface to which Ironic provision network to be connected @@ -429,11 +532,11 @@ the Ironic logs and baremetal operator to look at the state of servers. Openstack baremetal node shows all state of the server right from power, storage. -**Why provide network is required?** +**Why provider network (baremetal network configuration) is required?** -Generally, provider network DHCP servers in lab provide the router and -DNS server details. In some lab setup DHCP server don't provide this -information. +Generally, provider network DHCP servers in a lab provide the router +and DNS server details. In some labs, there is no DHCP server or the +DHCP server does not provide this information. # License diff --git a/deploy/metal3-vm/.gitignore b/deploy/metal3-vm/.gitignore index 22fc63a..7872a3f 100644 --- a/deploy/metal3-vm/.gitignore +++ b/deploy/metal3-vm/.gitignore @@ -2,8 +2,8 @@ logs ironic.env config_*.sh !config_example.sh -master-0*.yaml -worker-0*.yaml +master-*.json +worker-*.json *.bk *.tmp *.swp diff --git a/deploy/metal3-vm/03_launch_mgmt_cluster.sh b/deploy/metal3-vm/03_launch_mgmt_cluster.sh index 62ab294..7ebad78 100755 --- a/deploy/metal3-vm/03_launch_mgmt_cluster.sh +++ b/deploy/metal3-vm/03_launch_mgmt_cluster.sh @@ -60,22 +60,6 @@ function launch_baremetal_operator { kubectl apply -f $BMODIR/operator/no_ironic/operator.yaml -n metal3 } -function cloud_init_scripts { - cat << 'EOF' -write_files: -- path: /var/lib/cloud/scripts/per-boot/run_dhclient.sh - owner: root:root - permissions: '0777' - content: | - #!/usr/bin/env bash - set -xe - for intf in /sys/class/net/*; do - sudo ifconfig `basename $intf` up - sudo dhclient -nw `basename $intf` - done -EOF -} - # documentation for the values below may be found at # https://cloudinit.readthedocs.io/en/latest/topics/modules.html create_userdata() { @@ -100,7 +84,6 @@ create_userdata() { fi cat $HOME/.ssh/id_rsa.pub >> $name-userdata.yaml - cloud_init_scripts >> $name-userdata.yaml printf "\n" >> $name-userdata.yaml } @@ -119,10 +102,32 @@ EOF kubectl apply -n metal3 -f $name-user-data-credential.yaml } +create_networkdata() { + name="$1" + node_networkdata $name > $name-networkdata.json +} + +apply_networkdata_credential() { + name="$1" + cat < ./$name-network-data-credential.yaml +apiVersion: v1 +data: + networkData: $(base64 -w 0 $name-networkdata.json) +kind: Secret +metadata: + name: $name-network-data + namespace: metal3 +type: Opaque +EOF + kubectl apply -n metal3 -f $name-network-data-credential.yaml +} + function make_bm_hosts { while IFS=',' read -r name address user password mac; do create_userdata $name apply_userdata_credential $name + create_networkdata $name + apply_networkdata_credential $name GO111MODULE=auto go run "${BMOPATH}"/cmd/make-bm-worker/main.go \ -address "$address" \ -password "$password" \ @@ -135,6 +140,9 @@ function make_bm_hosts { printf "\n userData:" >> $name-bm-node.yaml printf "\n name: ""%s" "$name""-user-data" >> $name-bm-node.yaml printf "\n namespace: metal3" >> $name-bm-node.yaml + printf "\n networkData:" >> $name-bm-node.yaml + printf "\n name: ""%s" "$name""-network-data" >> $name-bm-node.yaml + printf "\n namespace: metal3" >> $name-bm-node.yaml printf "\n rootDeviceHints:" >> $name-bm-node.yaml printf "\n minSizeGigabytes: 48\n" >> $name-bm-node.yaml kubectl apply -f $name-bm-node.yaml -n metal3 diff --git a/deploy/metal3-vm/lib/common.sh b/deploy/metal3-vm/lib/common.sh index 4d1a699..a767358 100644 --- a/deploy/metal3-vm/lib/common.sh +++ b/deploy/metal3-vm/lib/common.sh @@ -122,3 +122,8 @@ function list_nodes { ] | @csv' | \ sed 's/"//g' } + +function node_networkdata { + name=$1 + cat $NODES_FILE | jq -r --arg name "$name" '.nodes[] | select(.name==$name) | .net' +} diff --git a/deploy/metal3-vm/vm-setup/roles/libvirt/templates/ironic_nodes.json.j2 b/deploy/metal3-vm/vm-setup/roles/libvirt/templates/ironic_nodes.json.j2 index 7040199..021f0cd 100644 --- a/deploy/metal3-vm/vm-setup/roles/libvirt/templates/ironic_nodes.json.j2 +++ b/deploy/metal3-vm/vm-setup/roles/libvirt/templates/ironic_nodes.json.j2 @@ -29,11 +29,29 @@ "properties": { "local_gb": "{{ flavors[node.flavor].disk }}", "cpu_arch": "{{ libvirt_arch }}" + }, + "net": { + "links": [ + {% for network in networks %} + { + "id": "{{ network.name }}_nic", + "ethernet_mac_address": "{{ node_mac_map.get(node.name).get(network.name) }}", + "type": "phy" + }{% if not loop.last %},{% endif %} + {% endfor %} + ], + "networks": [ + {% for network in networks %} + { + "id": "{{ network.name }}", + "link": "{{ network.name }}_nic", + "type": "ipv4_dhcp" + }{% if not loop.last %},{% endif %} + {% endfor %} + ], + "services": [] } - } - {% if not loop.last %} - , - {% endif %} + }{% if not loop.last %},{% endif %} {% endfor %} ] } diff --git a/deploy/metal3/scripts/01_metal3.sh b/deploy/metal3/scripts/01_metal3.sh index ac6a80b..2ad233c 100755 --- a/deploy/metal3/scripts/01_metal3.sh +++ b/deploy/metal3/scripts/01_metal3.sh @@ -134,6 +134,11 @@ EOF printf "\n" >> $name-userdata.yaml } +create_networkdata() { + name="$1" + node_networkdata $name > $name-networkdata.json +} + function launch_baremetal_operator { docker pull $IRONIC_BAREMETAL_IMAGE kubectl apply -f bmo/namespace/namespace.yaml @@ -169,30 +174,6 @@ write_files: sed -i~ "/^GRUB_CMDLINE_LINUX=/{h;s/\(=\".*\)\"/\1 ${kernel_parameters}\"/};\${x;/^$/{s//GRUB_CMDLINE_LINUX=\"${kernel_parameters}\"/;H};x}" "$grub_file" update-grub reboot -EOF - cat << 'EOF' -- path: /var/lib/cloud/scripts/per-boot/run_dhclient.sh - owner: root:root - permissions: '0777' - content: | - #!/usr/bin/env bash - set -xe - for intf in /sys/class/net/*; do - sudo ifconfig `basename $intf` up - sudo dhclient -nw `basename $intf` - done -EOF - cat << EOF -- path: /var/lib/cloud/scripts/per-boot/set_provider_network.sh - owner: root:root - permissions: '0777' - content: | - #!/usr/bin/env bash - set -xe - route add default gw $PROVIDER_NETWORK_GATEWAY - sed -i -e 's/^#DNS=.*/DNS=$PROVIDER_NETWORK_DNS/g' /etc/systemd/resolved.conf - systemctl daemon-reload - systemctl restart systemd-resolved EOF } @@ -211,10 +192,27 @@ EOF kubectl apply -n metal3 -f $name-user-data-credential.yaml } +apply_networkdata_credential() { + name="$1" + cat < ./$name-network-data-credential.yaml +apiVersion: v1 +data: + networkData: $(base64 -w 0 $name-networkdata.json) +kind: Secret +metadata: + name: $name-network-data + namespace: metal3 +type: Opaque +EOF + kubectl apply -n metal3 -f $name-network-data-credential.yaml +} + function make_bm_hosts { while IFS=',' read -r name ipmi_username ipmi_password ipmi_address os_username os_password os_image_name; do create_userdata $name $os_username $os_password apply_userdata_credential $name + create_networkdata $name + apply_networkdata_credential $name GO111MODULE=auto go run $GOPATH/src/github.com/metal3-io/baremetal-operator/cmd/make-bm-worker/main.go \ -address "ipmi://$ipmi_address" \ @@ -228,6 +226,9 @@ function make_bm_hosts { printf "\n userData:" >> $name-bm-node.yaml printf "\n name: ""%s" "$name""-user-data" >> $name-bm-node.yaml printf "\n namespace: metal3" >> $name-bm-node.yaml + printf "\n networkData:" >> $name-bm-node.yaml + printf "\n name: ""%s" "$name""-network-data" >> $name-bm-node.yaml + printf "\n namespace: metal3" >> $name-bm-node.yaml printf "\n rootDeviceHints:" >> $name-bm-node.yaml printf "\n minSizeGigabytes: 48\n" >> $name-bm-node.yaml kubectl apply -f $name-bm-node.yaml -n metal3 diff --git a/deploy/metal3/scripts/nodes.json.sample b/deploy/metal3/scripts/nodes.json.sample index 4332c3e..a451e49 100644 --- a/deploy/metal3/scripts/nodes.json.sample +++ b/deploy/metal3/scripts/nodes.json.sample @@ -11,6 +11,46 @@ "image_name": "bionic-server-cloudimg-amd64.img", "username": "ubuntu", "password": "mypasswd" + }, + "net": { + "links": [ + { + "id": "baremetal_nic", + "ethernet_mac_address": "00:1e:67:fe:f4:19", + "type": "phy" + }, + { + "id": "bootstrap_nic", + "ethernet_mac_address": "00:1e:67:f8:6a:40", + "type": "phy" + }, + { + "id": "provisioning_nic", + "ethernet_mac_address": "00:1e:67:fe:f4:1a", + "type": "phy" + } + ], + "networks": [ + { + "id": "baremetal", + "link": "baremetal_nic", + "type": "ipv4", + "ip_address": "10.10.110.22/24", + "gateway": "10.10.110.1", + "dns_nameservers": ["8.8.8.8"] + }, + { + "id": "provisioning", + "link": "provisioning_nic", + "type": "ipv4_dhcp" + }, + { + "id": "bootstrap", + "link": "bootstrap_nic", + "type": "ipv4_dhcp" + } + ], + "services": [] } }, { @@ -24,6 +64,46 @@ "image_name": "bionic-server-cloudimg-amd64.img", "username": "ubuntu", "password": "mypasswd" + }, + "net": { + "links": [ + { + "id": "baremetal_nic", + "ethernet_mac_address": "00:1e:67:f1:5b:90", + "type": "phy" + }, + { + "id": "bootstrap_nic", + "ethernet_mac_address": "00:1e:67:f8:69:80", + "type": "phy" + }, + { + "id": "provisioning_nic", + "ethernet_mac_address": "00:1e:67:f1:5b:91", + "type": "phy" + } + ], + "networks": [ + { + "id": "baremetal", + "link": "baremetal_nic", + "type": "ipv4", + "ip_address": "10.10.110.203/24", + "gateway": "10.10.110.1", + "dns_nameservers": ["8.8.8.8"] + }, + { + "id": "provisioning", + "link": "provisioning_nic", + "type": "ipv4_dhcp" + }, + { + "id": "bootstrap", + "link": "bootstrap_nic", + "type": "ipv4_dhcp" + } + ], + "services": [] } } ] diff --git a/env/lib/common.sh b/env/lib/common.sh index ad8a256..400dbd0 100755 --- a/env/lib/common.sh +++ b/env/lib/common.sh @@ -22,10 +22,6 @@ BS_DHCP_INTERFACE=${BS_DHCP_INTERFACE:-} BS_DHCP_INTERFACE_IP=${BS_DHCP_INTERFACE_IP:-} BS_DHCP_DIR=${BS_DHCP_DIR:-$DOWNLOAD_PATH/dhcp} -#User Provider Network configuration -PROVIDER_NETWORK_GATEWAY=${PROVIDER_NETWORK_GATEWAY:-} -PROVIDER_NETWORK_DNS=${PROVIDER_NETWORK_DNS:-} - #Ironic variables IRONIC_IMAGE=${IRONIC_IMAGE:-"integratedcloudnative/ironic:v1.0-icn"} IRONIC_INSPECTOR_IMAGE=${IRONIC_INSPECTOR_IMAGE:-"integratedcloudnative/ironic-inspector:v1.0-icn"} @@ -105,3 +101,14 @@ function list_nodes { sed 's/"//g' } +function node_networkdata { + name=$1 + + NODES_FILE="${IRONIC_DATA_DIR}/nodes.json" + + if [ ! -f "$NODES_FILE" ]; then + exit 1 + fi + + cat $NODES_FILE | jq -r --arg name "$name" '.nodes[] | select(.name==$name) | .net' +} diff --git a/user_config.sh b/user_config.sh index a6fac44..ac63063 100644 --- a/user_config.sh +++ b/user_config.sh @@ -10,7 +10,3 @@ export IRONIC_INTERFACE="enp4s0f3" #Ironic Metal3 setting for IPMI LAN Network export IRONIC_IPMI_INTERFACE="eno1" export IRONIC_IPMI_INTERFACE_IP="10.10.110.25" - -#User Network configuration -export PROVIDER_NETWORK_GATEWAY="10.10.110.1" -export PROVIDER_NETWORK_DNS="8.8.8.8"