From: Todd Malsbary <todd.malsbary@intel.com>
Date: Tue, 11 Jan 2022 16:34:40 +0000 (-0800)
Subject: Add docker registry mirror support
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=88d3e292a2c12513244dc534726387c6d76145e2;p=icn.git

Add docker registry mirror support

Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: Ie0be33106abd79ce2010491e0210feba9066c4c6
---

diff --git a/Makefile b/Makefile
index 3f30c51..3c61317 100644
--- a/Makefile
+++ b/Makefile
@@ -79,6 +79,7 @@ cluster: bmh_provision \
 	cluster_provision
 
 kud_bm_deploy_mini:
+	source user_config.sh && \
 	pushd $(KUD_PATH) && ./kud_bm_launch.sh minimal v1 && popd
 
 kud_bm_deploy:
diff --git a/Vagrantfile b/Vagrantfile
index 656c25c..8ab75ed 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -16,12 +16,17 @@ site = ENV['ICN_SITE'] || 'vm'
 # IPMI address) specified in the machine pool values.  IPMI in the
 # virtual environment is emulated by virtualbmc listening on the host.
 baremetal_cidr = nil
+registry_mirrors = nil
 Dir.glob("deploy/site/#{site}/*.yaml") do |file|
   YAML.load_stream(File.read(file)) do |document|
     values = document.fetch('spec', {}).fetch('values', {})
-    next if values['machineName'].nil? || values['bootMACAddress'].nil?
-    bmc_host = URI.parse(values['bmcAddress']).host
-    baremetal_cidr = "#{IPAddr.new(bmc_host).mask(24)}/24"
+    unless values['bmcAddress'].nil?
+      bmc_host = URI.parse(values['bmcAddress']).host
+      baremetal_cidr = "#{IPAddr.new(bmc_host).mask(24)}/24"
+    end
+    unless values['dockerRegistryMirrors'].nil?
+      registry_mirrors = values['dockerRegistryMirrors'].join(' ')
+    end
   end
 end
 if baremetal_cidr.nil?
@@ -102,7 +107,7 @@ Vagrant.configure("2") do |config|
 
     m.trigger.after [:up] do |trigger|
       trigger.name = 'Creating ICN user_config.sh'
-      trigger.run = {inline: "./tools/vagrant/create_user_config.sh"}
+      trigger.run = {inline: "bash -c 'DOCKER_REGISTRY_MIRRORS=\"#{registry_mirrors}\" ./tools/vagrant/create_user_config.sh'"}
     end
     m.vm.provision 'Configuring ICN prerequisites', type: 'shell', privileged: true, inline: <<-SHELL
       ssh-keygen -f "${HOME}/.ssh/id_rsa" -P "" <<<y
diff --git a/deploy/cluster/templates/_containerruntime.yaml b/deploy/cluster/templates/_containerruntime.yaml
index bd523bd..cf72b54 100644
--- a/deploy/cluster/templates/_containerruntime.yaml
+++ b/deploy/cluster/templates/_containerruntime.yaml
@@ -21,6 +21,9 @@
     mkdir -p /etc/containerd
     containerd config default | tee /etc/containerd/config.toml
     sed -i 's/\(.*\)\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.runc\.options\]/&\n\1  SystemdCgroup = true/' /etc/containerd/config.toml
+{{- if .dockerRegistryMirrors }}
+    sed -i 's/endpoint = \["https:\/\/registry-1\.docker\.io"\]/endpoint = {{ append .dockerRegistryMirrors "https://registry-1.docker.io" | toJson | replace "/" "\\/" | replace "[" "\\[" | replace "]" "\\]" }}/' /etc/containerd/config.toml
+{{- end }}
     systemctl restart containerd
 {{- else if eq .containerRuntime "docker" -}}
 - path: /usr/local/bin/install-container-runtime.sh
@@ -39,6 +42,9 @@
       "log-opts": {
         "max-size": "100m"
       },
+{{- if .dockerRegistryMirrors }}
+      "registry-mirrors": {{ .dockerRegistryMirrors | toJson }}
+{{- end }}
       "storage-driver": "overlay2"
     }
 {{- end }}
diff --git a/deploy/cluster/templates/kubeadmconfigtemplate.yaml b/deploy/cluster/templates/kubeadmconfigtemplate.yaml
index 33afc28..c2858d6 100644
--- a/deploy/cluster/templates/kubeadmconfigtemplate.yaml
+++ b/deploy/cluster/templates/kubeadmconfigtemplate.yaml
@@ -15,6 +15,9 @@ spec:
             kube-reserved: cpu=100m,memory=256Mi
           name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}'
       preKubeadmCommands:
+{{- if .Values.preKubeadmCommands }}
+{{ toYaml .Values.preKubeadmCommands | indent 6 }}
+{{- end }}
 {{- if .Values.networks }}
       # Without touching up /etc/hosts, kubeadm may pick the wrong
       # (i.e. provisioning network) address for the node IP
diff --git a/deploy/cluster/templates/kubeadmcontrolplane.yaml b/deploy/cluster/templates/kubeadmcontrolplane.yaml
index c960733..8d5e5e8 100644
--- a/deploy/cluster/templates/kubeadmcontrolplane.yaml
+++ b/deploy/cluster/templates/kubeadmcontrolplane.yaml
@@ -27,6 +27,9 @@ spec:
           kube-reserved: cpu=200m,memory=512Mi
         name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}'
     preKubeadmCommands:
+{{- if .Values.preKubeadmCommands }}
+{{ toYaml .Values.preKubeadmCommands | indent 4 }}
+{{- end }}
 {{- if .Values.networks }}
     # Without touching up /etc/hosts, kubeadm may pick the wrong
     # (i.e. provisioning network) address for the node IP
diff --git a/deploy/cluster/values.yaml b/deploy/cluster/values.yaml
index 63b7fad..499b667 100644
--- a/deploy/cluster/values.yaml
+++ b/deploy/cluster/values.yaml
@@ -104,3 +104,10 @@ kubeVersion: 1.21.6-00
 containerdVersion: 1.4.11-1
 # dockerVersion is the version of docker installed.
 dockerVersion: 5:20.10.10~3-0~ubuntu-focal
+# dockerRegistryMirrors are optional mirrors for docker.io, in priority order
+#dockerRegistryMirrors: ["https://myregistry.com:5000"]
+# preKubeadmCommands execute on the provisioned machine before the
+# container runtime or K8s is configured
+#preKubeadmCommands:
+#- apt-get update -y
+#- apt-get install -y unzip
diff --git a/deploy/kud/kud_bm_launch.sh b/deploy/kud/kud_bm_launch.sh
index 963e528..b7dc9b8 100755
--- a/deploy/kud/kud_bm_launch.sh
+++ b/deploy/kud/kud_bm_launch.sh
@@ -30,10 +30,18 @@ function set_ssh_key {
 function set_bm_kud {
     pushd ${KUDPATH}/kud/hosting_providers/vagrant/inventory
     HOST_IP=${HOST_IP:-$(hostname -I | cut -d ' ' -f 1)}
+    DOCKER_OPTIONS=""
+    if [[ ! -z "${DOCKER_REGISTRY_MIRRORS+x}" ]]; then
+	OPTIONS=""
+	for mirror in ${DOCKER_REGISTRY_MIRRORS}; do
+	    OPTIONS="${OPTIONS} --registry-mirror=${mirror}"
+	done
+	DOCKER_OPTIONS="docker_options=\"${OPTIONS# }\""
+    fi
     if [ "$1" == "minimal" ] ; then
         cat <<EOL > hosts.ini
 [all]
-$HOSTNAME ansible_ssh_host=${HOST_IP} ansible_ssh_port=22
+$HOSTNAME ansible_ssh_host=${HOST_IP} ansible_ssh_port=22 ${DOCKER_OPTIONS}
 
 [kube-master]
 $HOSTNAME
@@ -51,7 +59,7 @@ EOL
     else
         cat <<EOL > hosts.ini
 [all]
-$HOSTNAME ansible_ssh_host=${HOST_IP} ansible_ssh_port=22
+$HOSTNAME ansible_ssh_host=${HOST_IP} ansible_ssh_port=22 ${DOCKER_OPTIONS}
 
 [kube-master]
 $HOSTNAME
diff --git a/tools/vagrant/create_user_config.sh b/tools/vagrant/create_user_config.sh
index 33850c8..e753481 100755
--- a/tools/vagrant/create_user_config.sh
+++ b/tools/vagrant/create_user_config.sh
@@ -7,3 +7,11 @@ cat <<EOF >user_config.sh
 #Ironic Metal3 settings for provisioning network
 export IRONIC_INTERFACE="eth1"
 EOF
+
+if [[ ! -z "${DOCKER_REGISTRY_MIRRORS+x}" ]]; then
+    cat <<EOF >>user_config.sh
+
+#Use a registry mirror for downloading container images
+export DOCKER_REGISTRY_MIRRORS="${DOCKER_REGISTRY_MIRRORS}"
+EOF
+fi