From: Todd Malsbary Date: Fri, 19 Nov 2021 17:01:22 +0000 (-0800) Subject: Reduce machine, cluster charts to single instances X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F40%2F4540%2F1;p=icn.git Reduce machine, cluster charts to single instances The prior versions of the chart allow defining multiple instances of machines and clusters to be defined in a single release of the chart. This change now requires that each instance be defined as a unique release. While slightly more verbose intially, this simplifies documentation and allows default values to be provided in a sensible way. Signed-off-by: Todd Malsbary Change-Id: I9cd9694cf5de59439b393d5c9d271783d8c7ad54 --- diff --git a/.gitignore b/.gitignore index 1e695da..406e7be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,11 @@ deploy/ironic/logs/ deploy/baremetal-operator/logs/ deploy/cert-manager/logs/ -deploy/metal3/scripts/machines-values.yaml +deploy/metal3/scripts/machine-*-values.yaml deploy/cluster-api/logs/ -deploy/clusters/logs/ +deploy/cluster/logs/ env/metal3/logs/ -deploy/clusters/addons/ +deploy/cluster/addons/ build/ .vagrant/ deploy/kata/logs/ diff --git a/deploy/clusters/.helmignore b/deploy/cluster/.helmignore similarity index 100% rename from deploy/clusters/.helmignore rename to deploy/cluster/.helmignore diff --git a/deploy/cluster/Chart.yaml b/deploy/cluster/Chart.yaml new file mode 100644 index 0000000..e453e9b --- /dev/null +++ b/deploy/cluster/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: cluster +version: 0.1.0 +description: A Cluster API bare metal cluster +type: application +sources: +- https://gerrit.akraino.org/r/icn diff --git a/deploy/clusters/clusters.sh b/deploy/cluster/cluster.sh similarity index 84% rename from deploy/clusters/clusters.sh rename to deploy/cluster/cluster.sh index 73f63d8..7bbc46f 100755 --- a/deploy/clusters/clusters.sh +++ b/deploy/cluster/cluster.sh @@ -17,15 +17,13 @@ function build_source { # Flannel curl -sL https://raw.githubusercontent.com/coreos/flannel/${FLANNEL_VERSION}/Documentation/kube-flannel.yml -o ${SCRIPTDIR}/addons/flannel.yaml cat <${SCRIPTDIR}/templates/flannel-addon.yaml -{{- range \$clusterName, \$cluster := .Values.clusters }} -{{- if eq \$cluster.cni "flannel" }} +{{- if eq .Values.cni "flannel" }} --- $(kubectl create configmap flannel-addon --from-file=${SCRIPTDIR}/addons/flannel.yaml -o yaml --dry-run=client) {{- end }} -{{- end }} EOF - sed -i -e 's/ name: flannel-addon/ name: {{ $clusterName }}-flannel-addon/' ${SCRIPTDIR}/templates/flannel-addon.yaml - sed -i -e 's/10.244.0.0\/16/{{ $cluster.podCidr }}/' ${SCRIPTDIR}/templates/flannel-addon.yaml + sed -i -e 's/ name: flannel-addon/ name: {{ .Values.clusterName }}-flannel-addon/' ${SCRIPTDIR}/templates/flannel-addon.yaml + sed -i -e 's/10.244.0.0\/16/{{ .Values.podCidr }}/' ${SCRIPTDIR}/templates/flannel-addon.yaml # Flux flux install --export >${SCRIPTDIR}/addons/flux-system.yaml @@ -36,38 +34,36 @@ EOF apiVersion: source.toolkit.fluxcd.io/v1beta1 kind: GitRepository metadata: - name: {{ $cluster.flux.repositoryName }} + name: {{ .Values.flux.repositoryName }} namespace: flux-system spec: gitImplementation: go-git interval: 1m0s ref: - branch: {{ $cluster.flux.branch }} + branch: {{ .Values.flux.branch }} timeout: 20s - url: {{ $cluster.flux.url }} + url: {{ .Values.flux.url }} --- apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 kind: Kustomization metadata: - name: {{ $clusterName }}-flux-sync + name: {{ .Values.clusterName }}-flux-sync namespace: flux-system spec: interval: 10m0s - path: {{ $cluster.flux.path }} + path: {{ .Values.flux.path }} prune: true sourceRef: kind: GitRepository - name: {{ $cluster.flux.repositoryName }} + name: {{ .Values.flux.repositoryName }} EOF cat <${SCRIPTDIR}/templates/flux-addon.yaml -{{- range \$clusterName, \$cluster := .Values.clusters }} -{{- if \$cluster.flux }} +{{- if .Values.flux }} --- $(kubectl create configmap flux-addon --from-file=${SCRIPTDIR}/addons/flux-system.yaml,${SCRIPTDIR}/addons/sync.yaml -o yaml --dry-run=client) {{- end }} -{{- end }} EOF - sed -i -e 's/ name: flux-addon/ name: {{ $clusterName }}-flux-addon/' ${SCRIPTDIR}/templates/flux-addon.yaml + sed -i -e 's/ name: flux-addon/ name: {{ .Values.clusterName }}-flux-addon/' ${SCRIPTDIR}/templates/flux-addon.yaml # PodSecurityPolicy is being replaced in future versions of K8s. # The recommended practice is described by K8s at @@ -209,12 +205,10 @@ subjects: apiGroup: rbac.authorization.k8s.io EOF cat <${SCRIPTDIR}/templates/podsecurity-addon.yaml -{{- range \$clusterName, \$cluster := .Values.clusters }} --- $(kubectl create configmap podsecurity-addon --from-file=${SCRIPTDIR}/addons/podsecurity.yaml -o yaml --dry-run=client) -{{- end }} EOF - sed -i -e 's/ name: podsecurity-addon/ name: {{ $clusterName }}-podsecurity-addon/' ${SCRIPTDIR}/templates/podsecurity-addon.yaml + sed -i -e 's/ name: podsecurity-addon/ name: {{ .Values.clusterName }}-podsecurity-addon/' ${SCRIPTDIR}/templates/podsecurity-addon.yaml } diff --git a/deploy/clusters/resources/harden_k8s.sh b/deploy/cluster/resources/harden_k8s.sh similarity index 100% rename from deploy/clusters/resources/harden_k8s.sh rename to deploy/cluster/resources/harden_k8s.sh diff --git a/deploy/clusters/resources/harden_os.sh b/deploy/cluster/resources/harden_os.sh similarity index 100% rename from deploy/clusters/resources/harden_os.sh rename to deploy/cluster/resources/harden_os.sh diff --git a/deploy/clusters/resources/override.conf b/deploy/cluster/resources/override.conf similarity index 100% rename from deploy/clusters/resources/override.conf rename to deploy/cluster/resources/override.conf diff --git a/deploy/clusters/resources/set_kernel_cmdline.sh b/deploy/cluster/resources/set_kernel_cmdline.sh similarity index 100% rename from deploy/clusters/resources/set_kernel_cmdline.sh rename to deploy/cluster/resources/set_kernel_cmdline.sh diff --git a/deploy/cluster/templates/NOTES.txt b/deploy/cluster/templates/NOTES.txt new file mode 100644 index 0000000..ef234d7 --- /dev/null +++ b/deploy/cluster/templates/NOTES.txt @@ -0,0 +1,21 @@ +Thank you for installing the {{ .Chart.Name }} chart of Akraino ICN. + +Your release is named {{ .Release.Name }}. + +To learn more about this release, try: + + $ helm status {{ .Release.Name }} + $ helm get all {{ .Release.Name }} + +To get a brief description of this cluster and its resources, try: + + $ clusterctl describe cluster {{ .Values.clusterName }} + +To view the status of this cluster's control plane, try: + + $ kubectl get kubeadmcontrolplane {{ .Values.clusterName }} + +Once the first control plane node is up and running, retrieve this +cluster's kubeconfig with: + + $ clusterctl get kubeconfig {{ .Values.clusterName }} > {{ .Values.clusterName }}.kubeconfig diff --git a/deploy/clusters/templates/_containerruntime.yaml b/deploy/cluster/templates/_containerruntime.yaml similarity index 97% rename from deploy/clusters/templates/_containerruntime.yaml rename to deploy/cluster/templates/_containerruntime.yaml index 41318de..bd523bd 100644 --- a/deploy/clusters/templates/_containerruntime.yaml +++ b/deploy/cluster/templates/_containerruntime.yaml @@ -1,4 +1,4 @@ -{{- define "clusters.containerRuntime" -}} +{{- define "cluster.containerRuntime" -}} {{- if eq .containerRuntime "containerd" -}} - path: /usr/local/bin/install-container-runtime.sh permissions: '0777' diff --git a/deploy/clusters/templates/_keepalived.yaml b/deploy/cluster/templates/_keepalived.yaml similarity index 92% rename from deploy/clusters/templates/_keepalived.yaml rename to deploy/cluster/templates/_keepalived.yaml index 52d117d..06eba5a 100644 --- a/deploy/clusters/templates/_keepalived.yaml +++ b/deploy/cluster/templates/_keepalived.yaml @@ -1,4 +1,4 @@ -{{- define "clusters.keepalived" -}} +{{- define "cluster.keepalived" -}} {{- if .keepalived -}} - path: /etc/keepalived/keepalived.conf content: | diff --git a/deploy/clusters/templates/cluster.yaml b/deploy/cluster/templates/cluster.yaml similarity index 57% rename from deploy/clusters/templates/cluster.yaml rename to deploy/cluster/templates/cluster.yaml index bee2940..c507750 100644 --- a/deploy/clusters/templates/cluster.yaml +++ b/deploy/cluster/templates/cluster.yaml @@ -1,26 +1,24 @@ -{{- range $clusterName, $cluster := .Values.clusters }} --- apiVersion: cluster.x-k8s.io/v1alpha4 kind: Cluster metadata: labels: - cluster.x-k8s.io/cluster-name: {{ $clusterName }} - {{- toYaml $cluster.clusterLabels | nindent 4 }} - name: {{ $clusterName }} + cluster.x-k8s.io/cluster-name: {{ .Values.clusterName }} + {{- toYaml .Values.clusterLabels | nindent 4 }} + name: {{ .Values.clusterName }} spec: clusterNetwork: pods: cidrBlocks: - - {{ $cluster.podCidr }} + - {{ .Values.podCidr }} services: cidrBlocks: - 10.244.0.0/18 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 kind: KubeadmControlPlane - name: {{ $clusterName }} + name: {{ .Values.clusterName }} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 kind: Metal3Cluster - name: {{ $clusterName }} -{{- end }} + name: {{ .Values.clusterName }} diff --git a/deploy/cluster/templates/clusterresourceset.yaml b/deploy/cluster/templates/clusterresourceset.yaml new file mode 100644 index 0000000..294e7bd --- /dev/null +++ b/deploy/cluster/templates/clusterresourceset.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha4 +kind: ClusterResourceSet +metadata: + name: {{ .Values.clusterName }}-crs +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ .Values.clusterName }} + resources: +{{- if eq .Values.cni "flannel" }} + - name: {{ .Values.clusterName }}-flannel-addon + kind: ConfigMap +{{- end }} +{{- if .Values.flux }} + - name: {{ .Values.clusterName }}-flux-addon + kind: ConfigMap +{{- end }} + - name: {{ .Values.clusterName }}-podsecurity-addon + kind: ConfigMap diff --git a/deploy/clusters/templates/flannel-addon.yaml b/deploy/cluster/templates/flannel-addon.yaml similarity index 97% rename from deploy/clusters/templates/flannel-addon.yaml rename to deploy/cluster/templates/flannel-addon.yaml index ea560a4..d560501 100644 --- a/deploy/clusters/templates/flannel-addon.yaml +++ b/deploy/cluster/templates/flannel-addon.yaml @@ -1,5 +1,4 @@ -{{- range $clusterName, $cluster := .Values.clusters }} -{{- if eq $cluster.cni "flannel" }} +{{- if eq .Values.cni "flannel" }} --- apiVersion: v1 data: @@ -131,7 +130,7 @@ data: } net-conf.json: | { - "Network": "{{ $cluster.podCidr }}", + "Network": "{{ .Values.podCidr }}", "Backend": { "Type": "vxlan" } @@ -244,6 +243,5 @@ data: kind: ConfigMap metadata: creationTimestamp: null - name: {{ $clusterName }}-flannel-addon -{{- end }} + name: {{ .Values.clusterName }}-flannel-addon {{- end }} diff --git a/deploy/clusters/templates/flux-addon.yaml b/deploy/cluster/templates/flux-addon.yaml similarity index 99% rename from deploy/clusters/templates/flux-addon.yaml rename to deploy/cluster/templates/flux-addon.yaml index ad53ec4..81322e2 100644 --- a/deploy/clusters/templates/flux-addon.yaml +++ b/deploy/cluster/templates/flux-addon.yaml @@ -1,5 +1,4 @@ -{{- range $clusterName, $cluster := .Values.clusters }} -{{- if $cluster.flux }} +{{- if .Values.flux }} --- apiVersion: v1 data: @@ -4122,31 +4121,30 @@ data: apiVersion: source.toolkit.fluxcd.io/v1beta1 kind: GitRepository metadata: - name: {{ $cluster.flux.repositoryName }} + name: {{ .Values.flux.repositoryName }} namespace: flux-system spec: gitImplementation: go-git interval: 1m0s ref: - branch: {{ $cluster.flux.branch }} + branch: {{ .Values.flux.branch }} timeout: 20s - url: {{ $cluster.flux.url }} + url: {{ .Values.flux.url }} --- apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 kind: Kustomization metadata: - name: {{ $clusterName }}-flux-sync + name: {{ .Values.clusterName }}-flux-sync namespace: flux-system spec: interval: 10m0s - path: {{ $cluster.flux.path }} + path: {{ .Values.flux.path }} prune: true sourceRef: kind: GitRepository - name: {{ $cluster.flux.repositoryName }} + name: {{ .Values.flux.repositoryName }} kind: ConfigMap metadata: creationTimestamp: null - name: {{ $clusterName }}-flux-addon -{{- end }} + name: {{ .Values.clusterName }}-flux-addon {{- end }} diff --git a/deploy/clusters/templates/kubeadmconfigtemplate.yaml b/deploy/cluster/templates/kubeadmconfigtemplate.yaml similarity index 75% rename from deploy/clusters/templates/kubeadmconfigtemplate.yaml rename to deploy/cluster/templates/kubeadmconfigtemplate.yaml index 741bf69..c56ed0f 100644 --- a/deploy/clusters/templates/kubeadmconfigtemplate.yaml +++ b/deploy/cluster/templates/kubeadmconfigtemplate.yaml @@ -1,9 +1,8 @@ -{{- range $clusterName, $cluster := .Values.clusters }} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 kind: KubeadmConfigTemplate metadata: - name: {{ $clusterName }}-workers + name: {{ .Values.clusterName }}-workers spec: template: spec: @@ -16,10 +15,10 @@ spec: kube-reserved: cpu=100m,memory=256Mi name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}' preKubeadmCommands: -{{- if $cluster.networks }} +{{- if .Values.networks }} # Without touching up /etc/hosts, kubeadm may pick the wrong # (i.e. provisioning network) address for the node IP - - sed -i "1i $(ip -4 addr show dev {{ $cluster.networks.baremetal.interface }} | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1) $(hostname)" /etc/hosts + - sed -i "1i $(ip -4 addr show dev {{ .Values.networks.baremetal.interface }} | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1) $(hostname)" /etc/hosts {{- end }} - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" @@ -28,7 +27,7 @@ spec: - apt update -y - apt-get install -y ca-certificates - /usr/local/bin/install-container-runtime.sh - - apt-get install -y kubelet={{ $cluster.kubeVersion }} kubeadm={{ $cluster.kubeVersion }} kubectl={{ $cluster.kubeVersion }} + - apt-get install -y kubelet={{ .Values.kubeVersion }} kubeadm={{ .Values.kubeVersion }} kubectl={{ .Values.kubeVersion }} - systemctl enable --now kubelet postKubeadmCommands: - /usr/local/bin/harden_os.sh @@ -36,7 +35,7 @@ spec: # on files in /var/run, which won't persist after a reboot - /usr/local/bin/set_kernel_cmdline.sh files: -{{ include "clusters.containerRuntime" $cluster | indent 6 }} +{{ include "cluster.containerRuntime" .Values | indent 6 }} - path: /etc/systemd/system/containerd.service.d/override.conf content: | {{ $.Files.Get "resources/override.conf" | indent 10 }} @@ -49,15 +48,14 @@ spec: content: | {{ $.Files.Get "resources/set_kernel_cmdline.sh" | indent 10 }} users: - - name: {{ $cluster.userData.name }} + - name: {{ .Values.userData.name }} shell: /bin/bash lockPassword: False # Necessary to allow password login - passwd: {{ $cluster.userData.hashedPassword }} + passwd: {{ .Values.userData.hashedPassword }} sshAuthorizedKeys: - - {{ $cluster.userData.sshAuthorizedKey }} + - {{ .Values.userData.sshAuthorizedKey }} sudo: "ALL=(ALL) NOPASSWD:ALL" groups: sudo # Necessary to allow SSH logins (see /etc/ssh/sshd_config) - name: root sshAuthorizedKeys: - - {{ $cluster.userData.sshAuthorizedKey }} -{{- end }} + - {{ .Values.userData.sshAuthorizedKey }} diff --git a/deploy/clusters/templates/kubeadmcontrolplane.yaml b/deploy/cluster/templates/kubeadmcontrolplane.yaml similarity index 79% rename from deploy/clusters/templates/kubeadmcontrolplane.yaml rename to deploy/cluster/templates/kubeadmcontrolplane.yaml index 99c8bef..003144f 100644 --- a/deploy/clusters/templates/kubeadmcontrolplane.yaml +++ b/deploy/cluster/templates/kubeadmcontrolplane.yaml @@ -1,9 +1,8 @@ -{{- range $clusterName, $cluster := .Values.clusters }} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 kind: KubeadmControlPlane metadata: - name: {{ $clusterName }} + name: {{ .Values.clusterName }} spec: kubeadmConfigSpec: clusterConfiguration: @@ -25,10 +24,10 @@ spec: node-labels: metal3.io/uuid={{ "{{" }} ds.meta_data.uuid {{ "}}" }} name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}' preKubeadmCommands: -{{- if $cluster.networks }} +{{- if .Values.networks }} # Without touching up /etc/hosts, kubeadm may pick the wrong # (i.e. provisioning network) address for the node IP - - sed -i "1i $(ip -4 addr show dev {{ $cluster.networks.baremetal.interface }} | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1) $(hostname)" /etc/hosts + - sed -i "1i $(ip -4 addr show dev {{ .Values.networks.baremetal.interface }} | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1) $(hostname)" /etc/hosts {{- end }} - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" @@ -36,12 +35,12 @@ spec: - add-apt-repository "deb https://apt.kubernetes.io/ kubernetes-xenial main" - apt update -y - apt-get install -y ca-certificates -{{- if $cluster.keepalived }} +{{- if .Values.keepalived }} - apt-get install -y keepalived - systemctl enable --now keepalived {{- end }} - /usr/local/bin/install-container-runtime.sh - - apt-get install -y kubelet={{ $cluster.kubeVersion }} kubeadm={{ $cluster.kubeVersion }} kubectl={{ $cluster.kubeVersion }} + - apt-get install -y kubelet={{ .Values.kubeVersion }} kubeadm={{ .Values.kubeVersion }} kubectl={{ .Values.kubeVersion }} - systemctl enable --now kubelet postKubeadmCommands: - mkdir -p /home/ubuntu/.kube @@ -58,8 +57,8 @@ spec: # files in /var/run, which won't persist after a reboot - /usr/local/bin/set_kernel_cmdline.sh files: -{{ include "clusters.keepalived" $cluster | indent 4 }} -{{ include "clusters.containerRuntime" $cluster | indent 4 }} +{{ include "cluster.keepalived" .Values | indent 4 }} +{{ include "cluster.containerRuntime" .Values | indent 4 }} - path: /etc/systemd/system/containerd.service.d/override.conf content: | {{ $.Files.Get "resources/override.conf" | indent 8 }} @@ -76,27 +75,26 @@ spec: content: | {{ $.Files.Get "resources/set_kernel_cmdline.sh" | indent 8 }} users: - - name: {{ $cluster.userData.name }} + - name: {{ .Values.userData.name }} shell: /bin/bash lockPassword: False # Necessary to allow password login - passwd: {{ $cluster.userData.hashedPassword }} + passwd: {{ .Values.userData.hashedPassword }} sshAuthorizedKeys: - - {{ $cluster.userData.sshAuthorizedKey }} + - {{ .Values.userData.sshAuthorizedKey }} sudo: "ALL=(ALL) NOPASSWD:ALL" groups: sudo # Necessary to allow SSH logins (see /etc/ssh/sshd_config) - name: root sshAuthorizedKeys: - - {{ $cluster.userData.sshAuthorizedKey }} + - {{ .Values.userData.sshAuthorizedKey }} machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 kind: Metal3MachineTemplate - name: {{ $clusterName }}-controlplane + name: {{ .Values.clusterName }}-controlplane nodeDrainTimeout: 0s - replicas: {{ $cluster.numControlPlaneMachines }} + replicas: {{ .Values.numControlPlaneMachines }} rolloutStrategy: rollingUpdate: maxSurge: 1 type: RollingUpdate - version: {{ $cluster.k8sVersion }} -{{- end }} + version: {{ .Values.k8sVersion }} diff --git a/deploy/cluster/templates/machinedeployment.yaml b/deploy/cluster/templates/machinedeployment.yaml new file mode 100644 index 0000000..1398535 --- /dev/null +++ b/deploy/cluster/templates/machinedeployment.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ .Values.clusterName }} + name: {{ .Values.clusterName }} +spec: + clusterName: {{ .Values.clusterName }} + replicas: {{ .Values.numWorkerMachines }} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ .Values.clusterName }} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ .Values.clusterName }} + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 + kind: KubeadmConfigTemplate + name: {{ .Values.clusterName }}-workers + clusterName: {{ .Values.clusterName }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 + kind: Metal3MachineTemplate + name: {{ .Values.clusterName }}-workers + nodeDrainTimeout: 0s + version: {{ .Values.k8sVersion }} diff --git a/deploy/clusters/templates/metal3cluster.yaml b/deploy/cluster/templates/metal3cluster.yaml similarity index 52% rename from deploy/clusters/templates/metal3cluster.yaml rename to deploy/cluster/templates/metal3cluster.yaml index be52259..9daca3e 100644 --- a/deploy/clusters/templates/metal3cluster.yaml +++ b/deploy/cluster/templates/metal3cluster.yaml @@ -1,12 +1,10 @@ -{{- range $clusterName, $cluster := .Values.clusters }} --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 kind: Metal3Cluster metadata: - name: {{ $clusterName }} + name: {{ .Values.clusterName }} spec: controlPlaneEndpoint: - host: {{ $cluster.controlPlaneEndpoint }} + host: {{ .Values.controlPlaneEndpoint }} port: 6443 noCloudProvider: true -{{- end }} diff --git a/deploy/clusters/templates/metal3datatemplate.yaml b/deploy/cluster/templates/metal3datatemplate.yaml similarity index 58% rename from deploy/clusters/templates/metal3datatemplate.yaml rename to deploy/cluster/templates/metal3datatemplate.yaml index a1d3b7f..b675ac8 100644 --- a/deploy/clusters/templates/metal3datatemplate.yaml +++ b/deploy/cluster/templates/metal3datatemplate.yaml @@ -1,16 +1,15 @@ -{{- range $clusterName, $cluster := .Values.clusters }} -{{- if $cluster.networks }} +{{- if .Values.networks }} --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 kind: Metal3DataTemplate metadata: - name: {{ $clusterName }}-nodepool + name: {{ .Values.clusterName }}-nodepool spec: - clusterName: {{ $clusterName }} + clusterName: {{ .Values.clusterName }} networkData: links: ethernets: -{{- range $name, $network := $cluster.networks }} +{{- range $name, $network := .Values.networks }} - id: {{ $name }}_nic macAddress: fromHostInterface: {{ $network.interface }} @@ -18,9 +17,8 @@ spec: {{- end }} networks: ipv4DHCP: -{{- range $name, $network := $cluster.networks }} +{{- range $name, $network := .Values.networks }} - id: {{ $name }} link: {{ $name }}_nic {{- end }} {{- end }} -{{- end }} diff --git a/deploy/cluster/templates/metal3machinetemplate-controlplane.yaml b/deploy/cluster/templates/metal3machinetemplate-controlplane.yaml new file mode 100644 index 0000000..f50c259 --- /dev/null +++ b/deploy/cluster/templates/metal3machinetemplate-controlplane.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3MachineTemplate +metadata: + name: {{ .Values.clusterName }}-controlplane +spec: + nodeReuse: false + template: + spec: + automatedCleaningMode: metadata +{{- if .Values.controlPlaneHostSelector }} + hostSelector: + {{- toYaml .Values.controlPlaneHostSelector | nindent 8 }} +{{- end }} +{{- if .Values.networks }} + dataTemplate: + name: {{ .Values.clusterName }}-nodepool +{{- end }} + image: + checksum: http://172.22.0.1:6180/images/{{ .Values.imageName }}.md5sum + url: http://172.22.0.1:6180/images/{{ .Values.imageName }} diff --git a/deploy/cluster/templates/metal3machinetemplate-workers.yaml b/deploy/cluster/templates/metal3machinetemplate-workers.yaml new file mode 100644 index 0000000..2720bc4 --- /dev/null +++ b/deploy/cluster/templates/metal3machinetemplate-workers.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3MachineTemplate +metadata: + name: {{ .Values.clusterName }}-workers +spec: + nodeReuse: false + template: + spec: + automatedCleaningMode: metadata +{{- if .Values.workersHostSelector }} + hostSelector: + {{- toYaml .Values.workersHostSelector | nindent 8 }} +{{- end }} +{{- if .Values.networks }} + dataTemplate: + name: {{ .Values.clusterName }}-nodepool +{{- end }} + image: + checksum: http://172.22.0.1:6180/images/{{ .Values.imageName }}.md5sum + url: http://172.22.0.1:6180/images/{{ .Values.imageName }} diff --git a/deploy/clusters/templates/podsecurity-addon.yaml b/deploy/cluster/templates/podsecurity-addon.yaml similarity index 98% rename from deploy/clusters/templates/podsecurity-addon.yaml rename to deploy/cluster/templates/podsecurity-addon.yaml index 55b9607..592b706 100644 --- a/deploy/clusters/templates/podsecurity-addon.yaml +++ b/deploy/cluster/templates/podsecurity-addon.yaml @@ -1,4 +1,3 @@ -{{- range $clusterName, $cluster := .Values.clusters }} --- apiVersion: v1 data: @@ -297,5 +296,4 @@ data: kind: ConfigMap metadata: creationTimestamp: null - name: {{ $clusterName }}-podsecurity-addon -{{- end }} + name: {{ .Values.clusterName }}-podsecurity-addon diff --git a/deploy/cluster/values.yaml b/deploy/cluster/values.yaml new file mode 100644 index 0000000..5b0f8c4 --- /dev/null +++ b/deploy/cluster/values.yaml @@ -0,0 +1,108 @@ +# clusterName is the name of the cluster. +clusterName: cluster-1 + +# clusterLabels is a dictionary of labels. The provider and site +# labels are shown as examples only; additional labels may be +# provided. +clusterLabels: + provider: icn + site: test-site + +# numControlPlaneMachines is the number of control plane nodes. +numControlPlaneMachines: 1 + +# numWorkerMachines is the number of worker nodes. +numWorkerMachines: 1 + +# controlPlaneEndpoint is the address of the control plane endpoint. +# With a highly-available control plane this would typically be a +# load-balanced virtual IP, however other configurations are possible +# as shown below. +controlPlaneEndpoint: 192.168.151.254 +# controlPlanePrefix is the network mask of the control plane +# endpoint. +controlPlanePrefix: 24 + +# keepalived is one mechanism to provide a virtual control plane +# endpoint. keepalived uses the VRRP protocol to assign the control +# plane endpoint among the control plane nodes. +#keepalived: +# # The interface must be the same as the baremetal interface. +# interface: ens6 +# routerId: 3 + +# controlPlaneHostSelector uses labels added to the BareMetalHost +# resources to select specific machines for the control plane. Using +# this mechanism for example, one could assign a static address of a +# known machine to the control plane endpoint. +#controlPlaneHostSelector: +# matchLabels: +# machine: machine-1 +# controlPlaneHostSelector uses labels added to the BareMetalHost +# resources to select specific machines for workers. +#workersHostSelector: +# matchLabels: +# machine: machine-2 + +# networks can be used when DHCP is present and the network +# configuration of each machine may be provided with a template +# containing the names of the baremetal and provisioning interfaces. +#networks: +# baremetal: +# interface: ens6 +# provisioning: +# interface: ens5 + +# userData is used to provide cloud-init data for machines in the +# cluster. See +# https://cloudinit.readthedocs.io/en/latest/topics/modules.html#users-and-groups +# for more information. +userData: + name: ubuntu + # hashedPasswd was created with `mkpasswd --method=SHA-512 --rounds + # 10000 "mypasswd"`. + hashedPassword: $6$rounds=10000$PJLOBdyTv23pNp$9RpaAOcibbXUMvgJScKK2JRQioXW4XAVFMRKqgCB5jC4QmtAdbA70DU2jTcpAd6pRdEZIaWFjLCNQMBmiiL40. + # sshAuthorizedKey key will also be authorized to login as the root + # user. + sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump + +# flux provides bootstrapping configuration of the cluster. When +# enabled, the Flux controllers will be installed into the cluster and +# begin reconciling the resources located at the specified location. +flux: + repositoryName: icn + url: https://gerrit.akraino.org/r/icn + branch: master + # path is the repository to the resources to be applied to the + # cluster. + path: ./deploy/site/e2etest + +# containerRuntime may be containerd or docker. +containerRuntime: containerd + +# podCidr is the POD CIDR. +podCidr: 10.244.64.0/18 + +# cni is the cluster CNI. The only currently supported CNI is +# flannel. +cni: flannel + +# All the version info is captured in one block here. Care must +# be taken to ensure that the OS, Kubernetes, and CRI (containerd +# or docker) versions are all compatible. +# +# Refer to the below for further information: +# - https://github.com/kubernetes/kubernetes/blob/master/build/dependencies.yaml +# - https://download.docker.com/linux/ubuntu/dists/focal/stable/binary-amd64/Packages +# +# imageName is the OS image. +imageName: focal-server-cloudimg-amd64.img +# k8s is the version of Kubernetes installed. +k8sVersion: v1.21.6 +# kubeVersion is the version of the kubelet, kubeadm, and kubectl +# packages. +kubeVersion: 1.21.6-00 +# containerd is the version of containerd installed. +containerdVersion: 1.4.11-1 +# dockerVersion is the version of docker installed. +dockerVersion: 5:20.10.10~3-0~ubuntu-focal diff --git a/deploy/clusters/Chart.yaml b/deploy/clusters/Chart.yaml deleted file mode 100644 index 1d71d39..0000000 --- a/deploy/clusters/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v2 -name: clusters -type: application -version: 0.1.0 diff --git a/deploy/clusters/ha-dhcp-values.yaml b/deploy/clusters/ha-dhcp-values.yaml deleted file mode 100644 index e36ad63..0000000 --- a/deploy/clusters/ha-dhcp-values.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# The dictionary of clusters to create. -clusters: - # The cluster name. - ha-dhcp: - - # Example provider and site labels; additional labels may be - # provided. - clusterLabels: - provider: icn - site: test-site - - # The number of control plane nodes. - numControlPlaneMachines: 3 - - # The number of worker nodes. - numWorkerMachines: 2 - - # The control plane endpoint of the cluster. This is a virtual IP - # managed by keepalived. - controlPlaneEndpoint: 192.168.151.254 - controlPlanePrefix: 24 - - # keepalived uses the VRRP protocol to assign the control plane - # endpoint among the control plane nodes. - keepalived: - # This interface must be the same as the baremetal interface. - interface: ens6 - routerId: 3 - - # Since DHCP is present, the network configuration of each machine - # may be provided with a template containing the names of the - # baremetal and provisioning interfaces. - networks: - baremetal: - interface: ens6 - provisioning: - interface: ens5 - - # The user account created in all the machines. - userData: - name: ubuntu - # mkpasswd --method=SHA-512 --rounds 10000 "mypasswd" - hashedPassword: $6$rounds=10000$PJLOBdyTv23pNp$9RpaAOcibbXUMvgJScKK2JRQioXW4XAVFMRKqgCB5jC4QmtAdbA70DU2jTcpAd6pRdEZIaWFjLCNQMBmiiL40. - # This key will also be authorized to login as the root user - sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump - - # Flux bootstrapping of the cluster. When enabled, the Flux - # controllers will be installed into the cluster and begin - # reconciling the resources located at the specified location. - flux: - repositoryName: icn - url: https://gerrit.akraino.org/r/icn - branch: master - # The path at the repository to the resources to be applied to the - # cluster - path: ./deploy/test-site/ha-dhcp - - # The containerRuntime may be containerd or docker. - containerRuntime: containerd - - # The POD CIDR. - podCidr: 10.244.64.0/18 - - # The only currently supported CNI is flannel. - cni: flannel - - # All the version info is captured in one block here. Care must - # be taken to ensure that the OS, Kubernetes, and CRI (containerd - # or docker) versions are all compatible. - # - # Refer to the below for further information: - # - https://github.com/kubernetes/kubernetes/blob/master/build/dependencies.yaml - # - https://download.docker.com/linux/ubuntu/dists/focal/stable/binary-amd64/Packages - # - # The OS image. - imageName: focal-server-cloudimg-amd64.img - # The version of Kubernetes installed. - k8sVersion: v1.21.6 - # The version of the kubelet, kubeadm, and kubectl packages. - kubeVersion: 1.21.6-00 - # The version of the CRI installed. - containerdVersion: 1.4.11-1 - dockerVersion: 5:20.10.10~3-0~ubuntu-focal diff --git a/deploy/clusters/static-values.yaml b/deploy/clusters/static-values.yaml deleted file mode 100644 index f4ecfe0..0000000 --- a/deploy/clusters/static-values.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# The dictionary of clusters to create. -clusters: - # The cluster name. - static: - - # Example provider and site labels; additional labels may be - # provided. - clusterLabels: - provider: icn - site: test-site - - # The number of control plane nodes. - numControlPlaneMachines: 1 - - # The number of worker nodes. - numWorkerMachines: 1 - - # The control plane endpoint is set to the statically configured - # baremetal network address of node1. The host selector is - # defined below to ensure that node1 is the control plane. - controlPlaneEndpoint: 10.10.110.23 - controlPlanePrefix: 24 - - # Labels added to the BareMetalHost resources may be used here to - # select specific machines for control plane or workers. This is - # required here to ensure that node1 will host the control plane - # endpoint. - controlPlaneHostSelector: - matchLabels: - machine: node1 - workersHostSelector: - matchLabels: - machine: node2 - - # The user account created in all the machines. - userData: - name: ubuntu - # mkpasswd --method=SHA-512 --rounds 10000 "mypasswd" - hashedPassword: $6$rounds=10000$PJLOBdyTv23pNp$9RpaAOcibbXUMvgJScKK2JRQioXW4XAVFMRKqgCB5jC4QmtAdbA70DU2jTcpAd6pRdEZIaWFjLCNQMBmiiL40. - # This key will also be authorized to login as the root user - sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump - - # Flux bootstrapping of the cluster. When enabled, the Flux - # controllers will be installed into the cluster and begin - # reconciling the resources located at the specified location. - flux: - repositoryName: icn - url: https://gerrit.akraino.org/r/icn - branch: master - # The path at the repository to the resources to be applied to the - # cluster - path: ./deploy/test-site/static - - # The containerRuntime may be containerd or docker. - containerRuntime: containerd - - # The POD CIDR. - podCidr: 10.244.64.0/18 - - # The only currently supported CNI is flannel. - cni: flannel - - # All the version info is captured in one block here. Care must - # be taken to ensure that the OS, Kubernetes, and CRI (containerd - # or docker) versions are all compatible. - # - # Refer to the below for further information: - # - https://github.com/kubernetes/kubernetes/blob/master/build/dependencies.yaml - # - https://download.docker.com/linux/ubuntu/dists/focal/stable/binary-amd64/Packages - # - # The OS image. - imageName: focal-server-cloudimg-amd64.img - # The version of Kubernetes installed. - k8sVersion: v1.21.6 - # The version of the kubelet, kubeadm, and kubectl packages. - kubeVersion: 1.21.6-00 - # The version of the CRI installed. - containerdVersion: 1.4.11-1 - dockerVersion: 5:20.10.10~3-0~ubuntu-focal diff --git a/deploy/clusters/templates/clusterresourceset.yaml b/deploy/clusters/templates/clusterresourceset.yaml deleted file mode 100644 index d12e253..0000000 --- a/deploy/clusters/templates/clusterresourceset.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- range $clusterName, $cluster := .Values.clusters }} ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha4 -kind: ClusterResourceSet -metadata: - name: {{ $clusterName }}-crs -spec: - clusterSelector: - matchLabels: - cluster.x-k8s.io/cluster-name: {{ $clusterName }} - resources: -{{- if eq $cluster.cni "flannel" }} - - name: {{ $clusterName }}-flannel-addon - kind: ConfigMap -{{- end }} -{{- if $cluster.flux }} - - name: {{ $clusterName }}-flux-addon - kind: ConfigMap -{{- end }} - - name: {{ $clusterName }}-podsecurity-addon - kind: ConfigMap -{{- end }} diff --git a/deploy/clusters/templates/machinedeployment.yaml b/deploy/clusters/templates/machinedeployment.yaml deleted file mode 100644 index b294ffb..0000000 --- a/deploy/clusters/templates/machinedeployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- range $clusterName, $cluster := .Values.clusters }} ---- -apiVersion: cluster.x-k8s.io/v1alpha4 -kind: MachineDeployment -metadata: - labels: - cluster.x-k8s.io/cluster-name: {{ $clusterName }} - name: {{ $clusterName }} -spec: - clusterName: {{ $clusterName }} - replicas: {{ $cluster.numWorkerMachines }} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: {{ $clusterName }} - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: {{ $clusterName }} - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 - kind: KubeadmConfigTemplate - name: {{ $clusterName }}-workers - clusterName: {{ $clusterName }} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 - kind: Metal3MachineTemplate - name: {{ $clusterName }}-workers - nodeDrainTimeout: 0s - version: {{ $cluster.k8sVersion }} -{{- end }} diff --git a/deploy/clusters/templates/metal3machinetemplate-controlplane.yaml b/deploy/clusters/templates/metal3machinetemplate-controlplane.yaml deleted file mode 100644 index 6ba6ebf..0000000 --- a/deploy/clusters/templates/metal3machinetemplate-controlplane.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- range $clusterName, $cluster := .Values.clusters }} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 -kind: Metal3MachineTemplate -metadata: - name: {{ $clusterName }}-controlplane -spec: - nodeReuse: false - template: - spec: - automatedCleaningMode: metadata -{{- if $cluster.controlPlaneHostSelector }} - hostSelector: - {{- toYaml $cluster.controlPlaneHostSelector | nindent 8 }} -{{- end }} -{{- if $cluster.networks }} - dataTemplate: - name: {{ $clusterName }}-nodepool -{{- end }} - image: - checksum: http://172.22.0.1:6180/images/{{ $cluster.imageName }}.md5sum - url: http://172.22.0.1:6180/images/{{ $cluster.imageName }} -{{- end }} diff --git a/deploy/clusters/templates/metal3machinetemplate-workers.yaml b/deploy/clusters/templates/metal3machinetemplate-workers.yaml deleted file mode 100644 index bbb0dc5..0000000 --- a/deploy/clusters/templates/metal3machinetemplate-workers.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- range $clusterName, $cluster := .Values.clusters }} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 -kind: Metal3MachineTemplate -metadata: - name: {{ $clusterName }}-workers -spec: - nodeReuse: false - template: - spec: - automatedCleaningMode: metadata -{{- if $cluster.workersHostSelector }} - hostSelector: - {{- toYaml $cluster.workersHostSelector | nindent 8 }} -{{- end }} -{{- if $cluster.networks }} - dataTemplate: - name: {{ $clusterName }}-nodepool -{{- end }} - image: - checksum: http://172.22.0.1:6180/images/{{ $cluster.imageName }}.md5sum - url: http://172.22.0.1:6180/images/{{ $cluster.imageName }} -{{- end }} diff --git a/deploy/machines/.helmignore b/deploy/machine/.helmignore similarity index 100% rename from deploy/machines/.helmignore rename to deploy/machine/.helmignore diff --git a/deploy/machine/Chart.yaml b/deploy/machine/Chart.yaml new file mode 100644 index 0000000..cccac12 --- /dev/null +++ b/deploy/machine/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: machine +version: 0.1.0 +description: A Metal3 BareMetalHost +type: application +sources: +- https://gerrit.akraino.org/r/icn diff --git a/deploy/machine/templates/NOTES.txt b/deploy/machine/templates/NOTES.txt new file mode 100644 index 0000000..de617b9 --- /dev/null +++ b/deploy/machine/templates/NOTES.txt @@ -0,0 +1,8 @@ +Thank you for installing the {{ .Chart.Name }} chart of Akraino ICN. + +Your release is named {{ .Release.Name }}. + +To learn more about this release, try: + + $ helm status {{ .Release.Name }} + $ helm get all {{ .Release.Name }} diff --git a/deploy/machines/templates/_networkdata.json b/deploy/machine/templates/_networkdata.json similarity index 96% rename from deploy/machines/templates/_networkdata.json rename to deploy/machine/templates/_networkdata.json index ea407cc..f6f1901 100644 --- a/deploy/machines/templates/_networkdata.json +++ b/deploy/machine/templates/_networkdata.json @@ -1,4 +1,4 @@ -{{- define "machines.networkData" -}} +{{- define "machine.networkData" -}} {{- if .networks -}} { "links": [ diff --git a/deploy/machines/templates/_userdata.yaml b/deploy/machine/templates/_userdata.yaml similarity index 97% rename from deploy/machines/templates/_userdata.yaml rename to deploy/machine/templates/_userdata.yaml index 5f76ef6..e943e10 100644 --- a/deploy/machines/templates/_userdata.yaml +++ b/deploy/machine/templates/_userdata.yaml @@ -1,4 +1,4 @@ -{{- define "machines.userData" -}} +{{- define "machine.userData" -}} {{- if .userData -}} #cloud-config {{- if and .userData.name .userData.hashedPassword }} diff --git a/deploy/machine/templates/baremetalhost.yaml b/deploy/machine/templates/baremetalhost.yaml new file mode 100644 index 0000000..bf7d4a0 --- /dev/null +++ b/deploy/machine/templates/baremetalhost.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.machineName }}-bmc-secret +type: Opaque +data: + username: {{ .Values.bmcUsername | b64enc }} + password: {{ .Values.bmcPassword | b64enc }} +{{- if .Values.networks }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.machineName }}-network-data +type: Opaque +data: + networkData: {{ include "machine.networkData" .Values | b64enc }} +{{- end }} +{{- if .Values.userData }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.machineName }}-user-data +type: Opaque +data: + userData: {{ include "machine.userData" .Values | b64enc }} +{{- end }} +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: {{ .Values.machineName }} +{{- if .Values.machineLabels }} + labels: + {{- toYaml .Values.machineLabels | nindent 4 }} +{{- end }} +spec: + online: true +{{- if .Values.bootMACAddress }} + bootMACAddress: {{ .Values.bootMACAddress }} +{{- end }} + bmc: + address: {{ .Values.bmcAddress }} + credentialsName: {{ .Values.machineName }}-bmc-secret +{{- if .Values.networks }} + networkData: + name: {{ .Values.machineName }}-network-data + namespace: {{ .Release.Namespace }} +{{- end }} +{{- if .Values.userData }} + userData: + name: {{ .Values.machineName }}-user-data + namespace: {{ .Release.Namespace }} +{{- end }} + rootDeviceHints: + minSizeGigabytes: 48 +{{- if .Values.imageName }} + image: + url: http://172.22.0.1:6180/images/{{ .Values.imageName }} + checksum: http://172.22.0.1:6180/images/{{ .Values.imageName }}.md5sum +{{- end }} diff --git a/deploy/machine/values.yaml b/deploy/machine/values.yaml new file mode 100644 index 0000000..050fa61 --- /dev/null +++ b/deploy/machine/values.yaml @@ -0,0 +1,59 @@ +# machineName is the host name of the machine. +machineName: machine-1 + +# machineLabels is a dictionary of labels. The machine label is shown +# as an example only; additional labels may be provided. +#machineLabels: +# machine: machine-1 + +# bmcAddress is the address of the machine's bare metal controller. +bmcAddress: ipmi://10.10.110.11 +# bmcUsername is the username used to access the machine's bare metal +# controller. +bmcUsername: admin +# bmcPassword is the password used to access the machine's bare metal +# controller. +bmcPassword: password + +# bootMACAddress is the MAC address of the NIC that will PXE boot. +# This is only required when using virtual machines. +#bootMACAddress: 00:1e:67:fe:f4:1a + +# networks is used to provide per-machine network configuration. +# Where feasible (such as when DHCP is available or static IP pools +# can be used), the use of templates at the cluster level is +# encouraged instead. +#networks: +# baremetal: +# macAddress: 00:1e:67:fe:f4:19 +# # type is either ipv4 or ipv4_dhcp +# type: ipv4 +# # ipAddress is only valid for type ipv4 +# ipAddress: 10.10.110.21/24 +# # gateway is only valid for type ipv4 +# gateway: 10.10.110.1 +# # nameservers is an array of DNS servers; only valid for type ipv4 +# nameservers: ["8.8.8.8"] +# provisioning: +# macAddress: 00:1e:67:fe:f4:1a +# type: ipv4_dhcp +# private: +# macAddress: 00:1e:67:f8:6a:40 +# type: ipv4 +# ipAddress: 10.10.112.2/24 +# storage: +# macAddress: 00:1e:67:f8:6a:41 +# type: ipv4 +# ipAddress: 10.10.113.2/24 + +# userData is used to provide per-machine cloud-init data. Again, +# where feasible, the use of templates at the cluster level is +# encouraged instead. +#userData: +# name: ubuntu +# hashedPassword: $6$rounds=10000$PJLOBdyTv23pNp$9RpaAOcibbXUMvgJScKK2JRQioXW4XAVFMRKqgCB5jC4QmtAdbA70DU2jTcpAd6pRdEZIaWFjLCNQMBmiiL40. +# sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump +# fqdn: machine-1.akraino.icn.org + +# Legacy ICN support only, do not use with R6 or later +#imageName: focal-server-cloudimg-amd64.img diff --git a/deploy/machines/Chart.yaml b/deploy/machines/Chart.yaml deleted file mode 100644 index aa232fd..0000000 --- a/deploy/machines/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v2 -name: machines -type: application -version: 0.1.0 diff --git a/deploy/machines/example-values.yaml b/deploy/machines/example-values.yaml deleted file mode 100644 index 3138baa..0000000 --- a/deploy/machines/example-values.yaml +++ /dev/null @@ -1,27 +0,0 @@ -machines: - machine-1: - bmcUsername: admin - bmcPassword: password - bmcAddress: ipmi://192.168.151.1:6230 - - # Optional - bootMACAddress: 52:54:00:2b:bc:3a - - # Optional - imageName: focal-server-cloudimg-amd64.img - - # Optional - networks: - baremetal: - macAddress: 52:54:00:da:c9:7b - type: ipv4_dhcp - provisioning: - macAddress: 52:54:00:2b:bc:3a - type: ipv4_dhcp - - # Optional - userData: - name: ubuntu - hashedPassword: $6$rounds=10000$PJLOBdyTv23pNp$9RpaAOcibbXUMvgJScKK2JRQioXW4XAVFMRKqgCB5jC4QmtAdbA70DU2jTcpAd6pRdEZIaWFjLCNQMBmiiL40. - sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump - fqdn: machine-1.akraino.icn.org diff --git a/deploy/machines/templates/baremetalhost.yaml b/deploy/machines/templates/baremetalhost.yaml deleted file mode 100644 index 961380c..0000000 --- a/deploy/machines/templates/baremetalhost.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- range $name, $machine := .Values.machines }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $name }}-bmc-secret -type: Opaque -data: - username: {{ $machine.bmcUsername | b64enc }} - password: {{ $machine.bmcPassword | b64enc }} -{{- if $machine.networks }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $name }}-network-data -type: Opaque -data: - networkData: {{ include "machines.networkData" $machine | b64enc }} -{{- end }} -{{- if $machine.userData }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $name }}-user-data -type: Opaque -data: - userData: {{ include "machines.userData" $machine | b64enc }} -{{- end }} ---- -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - name: {{ $name }} -spec: - online: true -{{- if $machine.bootMACAddress }} - bootMACAddress: {{ $machine.bootMACAddress }} -{{- end }} - bmc: - address: {{ $machine.bmcAddress }} - credentialsName: {{ $name }}-bmc-secret -{{- if $machine.imageName }} - image: - url: http://172.22.0.1:6180/images/{{ $machine.imageName }} - checksum: http://172.22.0.1:6180/images/{{ $machine.imageName }}.md5sum -{{- end }} -{{- if $machine.networks }} - networkData: - name: {{ $name }}-network-data - namespace: {{ $.Release.Namespace }} -{{- end }} -{{- if $machine.userData }} - userData: - name: {{ $name }}-user-data - namespace: {{ $.Release.Namespace }} -{{- end }} - rootDeviceHints: - minSizeGigabytes: 48 -{{- end }} diff --git a/deploy/metal3/scripts/01_metal3.sh b/deploy/metal3/scripts/01_metal3.sh index add2a29..b3995a8 100755 --- a/deploy/metal3/scripts/01_metal3.sh +++ b/deploy/metal3/scripts/01_metal3.sh @@ -37,43 +37,46 @@ EOF exit 1 fi - printf " userData:\n" >>${SCRIPTDIR}/machines-values.yaml + printf "userData:\n" >>${SCRIPTDIR}/${name}-values.yaml if [ -n "$username" ]; then - printf " name: ${username}\n" >>${SCRIPTDIR}/machines-values.yaml + printf " name: ${username}\n" >>${SCRIPTDIR}/${name}-values.yaml fi if [ -n "$password" ]; then passwd=$(mkpasswd --method=SHA-512 --rounds 4096 "$password") - printf " hashedPassword: ${passwd}\n" >>${SCRIPTDIR}/machines-values.yaml + printf " hashedPassword: ${passwd}\n" >>${SCRIPTDIR}/${name}-values.yaml fi if [ -n "$COMPUTE_NODE_FQDN" ]; then - printf " fqdn: ${COMPUTE_NODE_FQDN}\n" >>${SCRIPTDIR}/machines-values.yaml + printf " fqdn: ${COMPUTE_NODE_FQDN}\n" >>${SCRIPTDIR}/${name}-values.yaml fi if [ ! -f $HOME/.ssh/id_rsa.pub ]; then yes y | ssh-keygen -t rsa -N "" -f $HOME/.ssh/id_rsa fi - printf " sshAuthorizedKey: $(cat $HOME/.ssh/id_rsa.pub)\n" >>${SCRIPTDIR}/machines-values.yaml + printf " sshAuthorizedKey: $(cat $HOME/.ssh/id_rsa.pub)\n" >>${SCRIPTDIR}/${name}-values.yaml } create_networkdata() { name="$1" - node_networkdata $name >>${SCRIPTDIR}/machines-values.yaml + node_networkdata $name >>${SCRIPTDIR}/${name}-values.yaml } function make_bm_hosts { while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do - printf " ${name}:\n" >>${SCRIPTDIR}/machines-values.yaml - printf " bmcUsername: ${ipmi_username}\n" >>${SCRIPTDIR}/machines-values.yaml - printf " bmcPassword: ${ipmi_password}\n" >>${SCRIPTDIR}/machines-values.yaml - printf " bmcAddress: ipmi://${ipmi_address}\n" >>${SCRIPTDIR}/machines-values.yaml + printf "machineName: ${name}\n" >${SCRIPTDIR}/${name}-values.yaml + printf "bmcUsername: ${ipmi_username}\n" >>${SCRIPTDIR}/${name}-values.yaml + printf "bmcPassword: ${ipmi_password}\n" >>${SCRIPTDIR}/${name}-values.yaml + printf "bmcAddress: ipmi://${ipmi_address}\n" >>${SCRIPTDIR}/${name}-values.yaml if [[ ! -z ${boot_mac} ]]; then - printf " bootMACAddress: ${boot_mac}\n" >>${SCRIPTDIR}/machines-values.yaml + printf "bootMACAddress: ${boot_mac}\n" >>${SCRIPTDIR}/${name}-values.yaml fi - printf " imageName: ${BM_IMAGE}\n" >>${SCRIPTDIR}/machines-values.yaml + printf "imageName: ${BM_IMAGE}\n" >>${SCRIPTDIR}/${name}-values.yaml create_userdata $name $os_username $os_password create_networkdata $name + + helm -n metal3 install ${name} ${SCRIPTDIR}/../../machine --create-namespace -f ${SCRIPTDIR}/${name}-values.yaml + done } @@ -94,18 +97,22 @@ function deprovision_bm_hosts { done } +function clean_bm_hosts { + while IFS=',' read -r name ipmi_username ipmi_password ipmi_address boot_mac os_username os_password os_image_name; do + helm -n metal3 uninstall ${name} + rm -rf ${SCRIPTDIR}/${name}-values.yaml + done +} + function clean_all { - helm -n metal3 uninstall machines - rm -f ${SCRIPTDIR}/machines-values.yaml + list_nodes | clean_bm_hosts if [ -f $IRONIC_DATA_DIR/nodes.json ]; then rm -rf $IRONIC_DATA_DIR/nodes.json fi } function apply_bm_hosts { - printf "machines:\n" >${SCRIPTDIR}/machines-values.yaml list_nodes | make_bm_hosts - helm -n metal3 install machines ${SCRIPTDIR}/../../machines --create-namespace -f ${SCRIPTDIR}/machines-values.yaml } function deprovision_all_hosts { diff --git a/deploy/site/vm/cluster-e2etest-values.yaml b/deploy/site/vm/cluster-e2etest-values.yaml new file mode 100644 index 0000000..9e399b0 --- /dev/null +++ b/deploy/site/vm/cluster-e2etest-values.yaml @@ -0,0 +1,15 @@ +clusterName: e2etest +clusterLabels: + site: vm +keepalived: + interface: ens6 + routerId: 3 +networks: + baremetal: + interface: ens6 + provisioning: + interface: ens5 +userData: + sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump +flux: + path: ./deploy/site/vm/e2etest diff --git a/deploy/site/vm/clusters-values.yaml b/deploy/site/vm/clusters-values.yaml deleted file mode 100644 index 5a8d277..0000000 --- a/deploy/site/vm/clusters-values.yaml +++ /dev/null @@ -1,34 +0,0 @@ -clusters: - e2etest: - clusterLabels: - provider: icn - site: vm - numControlPlaneMachines: 1 - numWorkerMachines: 1 - controlPlaneEndpoint: 192.168.151.254 - controlPlanePrefix: 24 - keepalived: - interface: ens6 - routerId: 3 - networks: - baremetal: - interface: ens6 - provisioning: - interface: ens5 - userData: - name: ubuntu - hashedPassword: $6$rounds=10000$PJLOBdyTv23pNp$9RpaAOcibbXUMvgJScKK2JRQioXW4XAVFMRKqgCB5jC4QmtAdbA70DU2jTcpAd6pRdEZIaWFjLCNQMBmiiL40. - sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrxu+fSrU51vgAO5zP5xWcTU8uLv4MkUZptE2m1BJE88JdQ80kz9DmUmq2AniMkVTy4pNeUW5PsmGJa+anN3MPM99CR9I37zRqy5i6rUDQgKjz8W12RauyeRMIBrbdy7AX1xasoTRnd6Ta47bP0egiFb+vUGnlTFhgfrbYfjbkJhVfVLCTgRw8Yj0NSK16YEyhYLbLXpix5udRpXSiFYIyAEWRCCsWJWljACr99P7EF82vCGI0UDGCCd/1upbUwZeTouD/FJBw9qppe6/1eaqRp7D36UYe3KzLpfHQNgm9AzwgYYZrD4tNN6QBMq/VUIuam0G1aLgG8IYRLs41HYkJ root@jump - flux: - repositoryName: icn - url: https://gerrit.akraino.org/r/icn - branch: master - path: ./deploy/site/vm/e2etest/ - containerRuntime: containerd - podCidr: 10.244.64.0/18 - cni: flannel - imageName: focal-server-cloudimg-amd64.img - k8sVersion: v1.21.6 - kubeVersion: 1.21.6-00 - containerdVersion: 1.4.11-1 - dockerVersion: 5:20.10.10~3-0~ubuntu-focal diff --git a/deploy/site/vm/vm.sh b/deploy/site/vm/vm.sh index 2d89d17..0d79743 100755 --- a/deploy/site/vm/vm.sh +++ b/deploy/site/vm/vm.sh @@ -13,17 +13,27 @@ function build { SSH_AUTHORIZED_KEY=$(cat ${HOME}/.ssh/id_rsa.pub) # Use ! instead of usual / to avoid escaping / in # SSH_AUTHORIZED_KEY - sed -e 's!sshAuthorizedKey: .*!sshAuthorizedKey: '"${SSH_AUTHORIZED_KEY}"'!' ${SCRIPTDIR}/clusters-values.yaml >${BUILDDIR}/clusters-values.yaml + sed -e 's!sshAuthorizedKey: .*!sshAuthorizedKey: '"${SSH_AUTHORIZED_KEY}"'!' ${SCRIPTDIR}/cluster-e2etest-values.yaml >${BUILDDIR}/cluster-e2etest-values.yaml +} + +function release_name { + local -r values_path=$1 + name=$(basename ${values_path}) + echo ${name%-values.yaml} } function deploy { - helm -n metal3 install machines ${SCRIPTDIR}/../../machines --create-namespace -f ${BUILDDIR}/machines-values.yaml - helm -n metal3 install clusters ${SCRIPTDIR}/../../clusters --create-namespace -f ${BUILDDIR}/clusters-values.yaml + for values in build/site/vm/machine-*-values.yaml; do + helm -n metal3 install $(release_name ${values}) ${SCRIPTDIR}/../../machine --create-namespace -f ${values} + done + helm -n metal3 install cluster-e2etest ${SCRIPTDIR}/../../cluster --create-namespace -f ${BUILDDIR}/cluster-e2etest-values.yaml } function clean { - helm -n metal3 uninstall clusters - helm -n metal3 uninstall machines + helm -n metal3 uninstall cluster-e2etest + for values in build/site/vm/machine-*-values.yaml; do + helm -n metal3 uninstall $(release_name ${values}) + done } function is_cluster_ready { diff --git a/env/lib/common.sh b/env/lib/common.sh index 38088c3..2009c58 100755 --- a/env/lib/common.sh +++ b/env/lib/common.sh @@ -147,7 +147,7 @@ function node_networkdata { exit 1 fi - printf " networks:\n" + printf "networks:\n" for network in $(cat $NODES_FILE | jq -r --arg name "$name" '.nodes[] | select(.name==$name) | .net.networks[].id'); do link=$(networkdata_networks_field $name $network "link") type=$(networkdata_networks_field $name $network "type") @@ -158,17 +158,17 @@ function node_networkdata { gateway=$(networkdata_networks_field $name $network "gateway") dns_nameservers=$(networkdata_networks_field $name $network "dns_nameservers") - printf " ${network}:\n" - printf " macAddress: ${mac}\n" - printf " type: ${type}\n" + printf " ${network}:\n" + printf " macAddress: ${mac}\n" + printf " type: ${type}\n" if [[ $ip_address != "null" ]]; then - printf " ipAddress: ${ip_address}\n" + printf " ipAddress: ${ip_address}\n" fi if [[ $gateway != "null" ]]; then - printf " gateway: ${gateway}\n" + printf " gateway: ${gateway}\n" fi if [[ $dns_nameservers != "null" ]]; then - printf " nameservers: ${dns_nameservers}\n" + printf " nameservers: ${dns_nameservers}\n" fi done } diff --git a/tools/vagrant/create_machine_resource.sh b/tools/vagrant/create_machine_resource.sh index 006ad53..e1afdc2 100755 --- a/tools/vagrant/create_machine_resource.sh +++ b/tools/vagrant/create_machine_resource.sh @@ -9,16 +9,11 @@ ipmi_host=$(virsh -c qemu:///system net-dumpxml ${site}-baremetal | xmlstarlet s ipmi_port=$((6230+index-1)) boot_mac=$(virsh -c qemu:///system dumpxml "${site}-${name}" | xmlstarlet sel -t -v "//interface[source/@network='${site}-provisioning']/mac/@address") -if [[ ${index} == 1 ]]; then - mkdir -p build/site/${site} - cat <build/site/${site}/machines-values.yaml -machines: -EOF -fi -cat <>build/site/${site}/machines-values.yaml - machine-${index}: - bootMACAddress: ${boot_mac} - bmcAddress: ipmi://${ipmi_host}:${ipmi_port} - bmcUsername: admin - bmcPassword: password +mkdir -p build/site/${site} +cat <build/site/${site}/machine-${index}-values.yaml +machineName: machine-${index} +bootMACAddress: ${boot_mac} +bmcAddress: ipmi://${ipmi_host}:${ipmi_port} +bmcUsername: admin +bmcPassword: password EOF