deploy/cluster/cluster.sh

   1 #!/usr/bin/env bash
   2 set -eux -o pipefail
   3
   4 SCRIPTDIR="$(readlink -f $(dirname ${BASH_SOURCE[0]}))"
   5 LIBDIR="$(dirname $(dirname ${SCRIPTDIR}))/env/lib"
   6
   7 source $LIBDIR/logging.sh
   8 source $LIBDIR/common.sh
   9
  10 function build_source_flannel {
  11     curl -sL https://raw.githubusercontent.com/coreos/flannel/${FLANNEL_VERSION}/Documentation/kube-flannel.yml -o ${SCRIPTDIR}/addons/flannel.yaml
  12     cat <<EOF >${SCRIPTDIR}/templates/flannel-addon.yaml
  13 {{- if eq .Values.cni "flannel" }}
  14 ---
  15 $(kubectl create configmap flannel-addon --from-file=${SCRIPTDIR}/addons/flannel.yaml -o yaml --dry-run=client)
  16 {{- end }}
  17 EOF
  18     sed -i -e 's/  name: flannel-addon/  name: {{ .Values.clusterName }}-flannel-addon/' ${SCRIPTDIR}/templates/flannel-addon.yaml
  19     sed -i -e 's/10.244.0.0\/16/{{ .Values.podCidr }}/' ${SCRIPTDIR}/templates/flannel-addon.yaml
  20 }
  21
  22 function build_source_flux {
  23     # NOTE: This reaches outside this directory to
  24     # deploy/site/cluster-addons/flux-system.  This is to ensure that
  25     # the day-0 config of a cluster using deploy/site/cluster-addons
  26     # is in sync with the chart.
  27     flux install --export >${SCRIPTDIR}/../site/cluster-addons/flux-system/gotk-components.yaml
  28     kustomize build ${SCRIPTDIR}/../site/cluster-addons/flux-system >${SCRIPTDIR}/addons/flux-system.yaml
  29     cat <<EOF >>${SCRIPTDIR}/addons/flux-system.yaml
  30 ---
  31 apiVersion: rbac.authorization.k8s.io/v1
  32 kind: RoleBinding
  33 metadata:
  34   name: psp:privileged:flux-system
  35   namespace: flux-system
  36 roleRef:
  37   apiGroup: rbac.authorization.k8s.io
  38   kind: ClusterRole
  39   name: psp:privileged
  40 subjects:
  41 - kind: Group
  42   name: system:serviceaccounts:flux-system
  43   apiGroup: rbac.authorization.k8s.io
  44 EOF
  45     # The name "sync" must be sorted after "flux-system" to ensure
  46     # CRDs are instantiated first
  47     cat <<'EOF' >${SCRIPTDIR}/addons/sync.yaml
  48 {{- if .Values.flux.decryptionSecret }}
  49 ---
  50 apiVersion: v1
  51 type: Opaque
  52 kind: Secret
  53 metadata:
  54   name: {{ .Values.flux.repositoryName }}-{{ .Values.flux.branch }}-sops-gpg
  55   namespace: flux-system
  56 data:
  57   sops.asc: {{ .Values.flux.decryptionSecret | b64enc }}
  58 {{- end }}
  59 ---
  60 apiVersion: source.toolkit.fluxcd.io/v1beta1
  61 kind: GitRepository
  62 metadata:
  63   name: {{ .Values.flux.repositoryName }}
  64   namespace: flux-system
  65 spec:
  66   gitImplementation: go-git
  67   interval: 1m0s
  68   ref:
  69     branch: {{ .Values.flux.branch }}
  70   timeout: 20s
  71   url: {{ .Values.flux.url }}
  72 ---
  73 apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
  74 kind: Kustomization
  75 metadata:
  76   name: {{ .Values.clusterName }}-flux-sync
  77   namespace: flux-system
  78 spec:
  79   interval: 10m0s
  80   path: {{ .Values.flux.path }}
  81   prune: true
  82   sourceRef:
  83     kind: GitRepository
  84     name: {{ .Values.flux.repositoryName }}
  85 {{- if .Values.flux.decryptionSecret }}
  86   decryption:
  87     provider: sops
  88     secretRef:
  89       name: {{ .Values.flux.repositoryName }}-{{ .Values.flux.branch }}-sops-gpg
  90 {{- end }}
  91 EOF
  92     cat <<EOF >${SCRIPTDIR}/templates/flux-addon.yaml
  93 {{- if .Values.flux }}
  94 ---
  95 $(kubectl create configmap flux-addon --from-file=${SCRIPTDIR}/addons/flux-system.yaml,${SCRIPTDIR}/addons/sync.yaml -o yaml --dry-run=client)
  96 {{- end }}
  97 EOF
  98     sed -i -e 's/  name: flux-addon/  name: {{ .Values.clusterName }}-flux-addon/' ${SCRIPTDIR}/templates/flux-addon.yaml
  99 }
 100
 101 function build_source_podsecurity {
 102     # PodSecurityPolicy is being replaced in future versions of K8s.
 103     # The recommended practice is described by K8s at
 104     # - https://kubernetes.io/docs/concepts/policy/pod-security-policy/#recommended-practice
 105     # - https://kubernetes.io/docs/concepts/security/pod-security-standards/
 106     # and provides three levels: privileged, baseline, and restricted.
 107     #
 108     # The question to answer here is how to reconcile the K8s levels
 109     # against the Akraino security requirements.
 110     #
 111     # For the time being, the below populates the cluster with the K8s
 112     # recommended levels and provides an additional policy (icn) bound
 113     # to the system:authenticated group to meet the Akraino
 114     # requirements.
 115     cat <<EOF >${SCRIPTDIR}/addons/podsecurity.yaml
 116 ---
 117 $(curl -sL https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/policy/privileged-psp.yaml)
 118 ---
 119 $(curl -sL https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/policy/baseline-psp.yaml)
 120 ---
 121 $(curl -sL https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/policy/restricted-psp.yaml)
 122 ---
 123 $(curl -sL https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/policy/privileged-psp.yaml |
 124   sed -e 's/  name: privileged/  name: icn/' |
 125   sed -e '/^  allowedCapabilities:/,/^  [!-]/d')
 126   allowedCapabilities:
 127     - 'NET_ADMIN'
 128     - 'SYS_ADMIN'
 129     - 'SYS_NICE'
 130     - 'SYS_PTRACE'
 131   requiredDropCapabilities:
 132     - 'NET_RAW'
 133 ---
 134 apiVersion: rbac.authorization.k8s.io/v1
 135 kind: ClusterRole
 136 metadata:
 137   name: psp:privileged
 138   labels:
 139     addonmanager.kubernetes.io/mode: Reconcile
 140 rules:
 141 - apiGroups:
 142   - policy
 143   resourceNames:
 144   - privileged
 145   resources:
 146   - podsecuritypolicies
 147   verbs:
 148   - use
 149 ---
 150 apiVersion: rbac.authorization.k8s.io/v1
 151 kind: ClusterRole
 152 metadata:
 153   name: psp:baseline
 154   labels:
 155     addonmanager.kubernetes.io/mode: Reconcile
 156 rules:
 157 - apiGroups:
 158   - policy
 159   resourceNames:
 160   - baseline
 161   resources:
 162   - podsecuritypolicies
 163   verbs:
 164   - use
 165 ---
 166 apiVersion: rbac.authorization.k8s.io/v1
 167 kind: ClusterRole
 168 metadata:
 169   name: psp:icn
 170   labels:
 171     addonmanager.kubernetes.io/mode: Reconcile
 172 rules:
 173 - apiGroups:
 174   - policy
 175   resourceNames:
 176   - icn
 177   resources:
 178   - podsecuritypolicies
 179   verbs:
 180   - use
 181 ---
 182 apiVersion: rbac.authorization.k8s.io/v1
 183 kind: ClusterRole
 184 metadata:
 185   name: psp:restricted
 186   labels:
 187     addonmanager.kubernetes.io/mode: Reconcile
 188 rules:
 189 - apiGroups:
 190   - policy
 191   resourceNames:
 192   - restricted
 193   resources:
 194   - podsecuritypolicies
 195   verbs:
 196   - use
 197 ---
 198 apiVersion: rbac.authorization.k8s.io/v1
 199 kind: RoleBinding
 200 metadata:
 201   name: psp:privileged:nodes
 202   namespace: kube-system
 203   labels:
 204     addonmanager.kubernetes.io/mode: Reconcile
 205 roleRef:
 206   apiGroup: rbac.authorization.k8s.io
 207   kind: ClusterRole
 208   name: psp:privileged
 209 subjects:
 210 - kind: Group
 211   name: system:nodes
 212   apiGroup: rbac.authorization.k8s.io
 213 ---
 214 apiVersion: rbac.authorization.k8s.io/v1
 215 kind: RoleBinding
 216 metadata:
 217   name: psp:privileged:kube-system
 218   namespace: kube-system
 219 roleRef:
 220   apiGroup: rbac.authorization.k8s.io
 221   kind: ClusterRole
 222   name: psp:privileged
 223 subjects:
 224 - kind: Group
 225   name: system:serviceaccounts:kube-system
 226   apiGroup: rbac.authorization.k8s.io
 227 ---
 228 apiVersion: rbac.authorization.k8s.io/v1
 229 kind: ClusterRoleBinding
 230 metadata:
 231   name: psp:icn:any
 232 roleRef:
 233   kind: ClusterRole
 234   name: psp:icn
 235   apiGroup: rbac.authorization.k8s.io
 236 subjects:
 237 - kind: Group
 238   name: system:authenticated
 239   apiGroup: rbac.authorization.k8s.io
 240 EOF
 241     cat <<EOF >${SCRIPTDIR}/templates/podsecurity-addon.yaml
 242 ---
 243 $(kubectl create configmap podsecurity-addon --from-file=${SCRIPTDIR}/addons/podsecurity.yaml -o yaml --dry-run=client)
 244 EOF
 245     sed -i -e 's/  name: podsecurity-addon/  name: {{ .Values.clusterName }}-podsecurity-addon/' ${SCRIPTDIR}/templates/podsecurity-addon.yaml
 246 }
 247
 248 function build_source_calico {
 249     mkdir -p ${SCRIPTDIR}/addons/calico/{base,ipv4,dualstack,ipv6}
 250     curl -sL https://docs.projectcalico.org/archive/${CALICO_VERSION%.*}/manifests/calico.yaml -o ${SCRIPTDIR}/addons/calico/base/calico.yaml
 251     # Remove trailing whitespace so that kubectl create configmap
 252     # doesn't insert explicit newlines
 253     sed -i -r 's/\s+$//g' ${SCRIPTDIR}/addons/calico/base/calico.yaml
 254     pushd ${SCRIPTDIR}/addons/calico/base && rm -f kustomization.yaml && kustomize create --autodetect && popd
 255
 256     # IPv4 only (the default)
 257     cat <<EOF >${SCRIPTDIR}/addons/calico/ipv4/ip-autodetection-method-patch.yaml
 258 kind: DaemonSet
 259 apiVersion: apps/v1
 260 metadata:
 261   name: calico-node
 262   namespace: kube-system
 263 spec:
 264   template:
 265     spec:
 266       containers:
 267         - name: calico-node
 268           env:
 269             - name: IP_AUTODETECTION_METHOD
 270               value: can-reach=www.google.com
 271 EOF
 272     cat <<EOF >${SCRIPTDIR}/addons/calico/ipv4/kustomization.yaml
 273 resources:
 274 - ../base
 275 patches:
 276 - path: ip-autodetection-method-patch.yaml
 277 EOF
 278     kustomize build ${SCRIPTDIR}/addons/calico/ipv4 >${SCRIPTDIR}/addons/calico/ipv4.yaml
 279     # Dual stack
 280     cat <<'EOF' >${SCRIPTDIR}/addons/calico/dualstack/configmap-patch.yaml
 281 kind: ConfigMap
 282 apiVersion: v1
 283 metadata:
 284   name: calico-config
 285   namespace: kube-system
 286 data:
 287   cni_network_config: |-
 288     {
 289       "name": "k8s-pod-network",
 290       "cniVersion": "0.3.1",
 291       "plugins": [
 292         {
 293           "type": "calico",
 294           "log_level": "info",
 295           "log_file_path": "/var/log/calico/cni/cni.log",
 296           "datastore_type": "kubernetes",
 297           "nodename": "__KUBERNETES_NODE_NAME__",
 298           "mtu": __CNI_MTU__,
 299           "ipam": {
 300               "type": "calico-ipam",
 301               "assign_ipv4": "true",
 302               "assign_ipv6": "true"
 303           },
 304           "policy": {
 305               "type": "k8s"
 306           },
 307           "kubernetes": {
 308               "kubeconfig": "__KUBECONFIG_FILEPATH__"
 309           }
 310         },
 311         {
 312           "type": "portmap",
 313           "snat": true,
 314           "capabilities": {"portMappings": true}
 315         },
 316         {
 317           "type": "bandwidth",
 318           "capabilities": {"bandwidth": true}
 319         }
 320       ]
 321     }
 322 EOF
 323     cat <<EOF >${SCRIPTDIR}/addons/calico/dualstack/ip-autodetection-method-patch.yaml
 324 kind: DaemonSet
 325 apiVersion: apps/v1
 326 metadata:
 327   name: calico-node
 328   namespace: kube-system
 329 spec:
 330   template:
 331     spec:
 332       containers:
 333         - name: calico-node
 334           env:
 335             - name: IP_AUTODETECTION_METHOD
 336               value: can-reach=www.google.com
 337             - name: IP6_AUTODETECTION_METHOD
 338               value: can-reach=www.google.com
 339             - name: IP6
 340               value: autodetect
 341             - name: FELIX_IPV6SUPPORT
 342               value: true
 343 EOF
 344     cat <<EOF >${SCRIPTDIR}/addons/calico/dualstack/kustomization.yaml
 345 resources:
 346 - ../base
 347 patches:
 348 - path: configmap-patch.yaml
 349 - path: ip-autodetection-method-patch.yaml
 350 EOF
 351     kustomize build ${SCRIPTDIR}/addons/calico/dualstack >${SCRIPTDIR}/addons/calico/dualstack.yaml
 352     # IPv6 only
 353     cat <<'EOF' >${SCRIPTDIR}/addons/calico/ipv6/configmap-patch.yaml
 354 kind: ConfigMap
 355 apiVersion: v1
 356 metadata:
 357   name: calico-config
 358   namespace: kube-system
 359 data:
 360   cni_network_config: |-
 361     {
 362       "name": "k8s-pod-network",
 363       "cniVersion": "0.3.1",
 364       "plugins": [
 365         {
 366           "type": "calico",
 367           "log_level": "info",
 368           "log_file_path": "/var/log/calico/cni/cni.log",
 369           "datastore_type": "kubernetes",
 370           "nodename": "__KUBERNETES_NODE_NAME__",
 371           "mtu": __CNI_MTU__,
 372           "ipam": {
 373               "type": "calico-ipam",
 374               "assign_ipv4": "false",
 375               "assign_ipv6": "true"
 376           },
 377           "policy": {
 378               "type": "k8s"
 379           },
 380           "kubernetes": {
 381               "kubeconfig": "__KUBECONFIG_FILEPATH__"
 382           }
 383         },
 384         {
 385           "type": "portmap",
 386           "snat": true,
 387           "capabilities": {"portMappings": true}
 388         },
 389         {
 390           "type": "bandwidth",
 391           "capabilities": {"bandwidth": true}
 392         }
 393       ]
 394     }
 395 EOF
 396     cat <<EOF >${SCRIPTDIR}/addons/calico/ipv6/ip-autodetection-method-patch.yaml
 397 kind: DaemonSet
 398 apiVersion: apps/v1
 399 metadata:
 400   name: calico-node
 401   namespace: kube-system
 402 spec:
 403   template:
 404     spec:
 405       containers:
 406         - name: calico-node
 407           env:
 408             - name: IP6_AUTODETECTION_METHOD
 409               value: can-reach=www.google.com
 410             - name: IP6
 411               value: autodetect
 412             - name: FELIX_IPV6SUPPORT
 413               value: true
 414             - name: IP
 415               value: none
 416             - name: CALICO_ROUTER_ID
 417               value: hash
 418 EOF
 419     cat <<EOF >${SCRIPTDIR}/addons/calico/ipv6/kustomization.yaml
 420 resources:
 421 - ../base
 422 patches:
 423 - path: configmap-patch.yaml
 424 - path: ip-autodetection-method-patch.yaml
 425 EOF
 426     kustomize build ${SCRIPTDIR}/addons/calico/ipv6 >${SCRIPTDIR}/addons/calico/ipv6.yaml
 427
 428     cat <<EOF >${SCRIPTDIR}/templates/calico-addon.yaml
 429 {{- if eq .Values.cni "calico" }}
 430 {{- if eq .Values.ipam "ipv4" }}
 431 ---
 432 $(kubectl create configmap calico-addon --from-file=calico.yaml=${SCRIPTDIR}/addons/calico/ipv4.yaml -o yaml --dry-run=client)
 433 {{- end }}
 434 {{- if eq .Values.ipam "dualstack" }}
 435 ---
 436 $(kubectl create configmap calico-addon --from-file=calico.yaml=${SCRIPTDIR}/addons/calico/dualstack.yaml -o yaml --dry-run=client)
 437 {{- end }}
 438 {{- if eq .Values.ipam "ipv6" }}
 439 ---
 440 $(kubectl create configmap calico-addon --from-file=calico.yaml=${SCRIPTDIR}/addons/calico/ipv6.yaml -o yaml --dry-run=client)
 441 {{- end }}
 442 {{- end }}
 443 EOF
 444     sed -i -e 's/  name: calico-addon/  name: {{ .Values.clusterName }}-calico-addon/' ${SCRIPTDIR}/templates/calico-addon.yaml
 445 }
 446
 447 # This may be used to update the in-place addon YAML files from the
 448 # upstream projects
 449 function build_source {
 450     mkdir -p ${SCRIPTDIR}/addons
 451     build_source_calico
 452     build_source_flannel
 453     build_source_flux
 454     build_source_podsecurity
 455 }
 456
 457 case $1 in
 458     "build-source") build_source ;;
 459     "foo") build_source_calico ;;
 460     *) cat <<EOF
 461 Usage: $(basename $0) COMMAND
 462
 463 Commands:
 464   build-source  - Rebuild the in-tree addon YAML files
 465 EOF
 466        ;;
 467 esac