From: Jianlin Lv Date: Fri, 7 Feb 2020 06:25:52 +0000 (+0800) Subject: Integrate cilium CNI into IEC X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F26%2F2226%2F5;p=iec.git Integrate cilium CNI into IEC Support cilium deployment on master and work nodes 1. Implement support for cilium CNI in setup-cni.sh; 2. Add work node initialization function for cilium; 3. Add cilium CNI deployment scripts and yaml file. Signed-off-by: Jianlin Lv Change-Id: Ief29cba051b0f5bb45ff3f3acbbbe0db50524cc5 --- diff --git a/src/foundation/scripts/cni/cilium/cilium_install.sh b/src/foundation/scripts/cni/cilium/cilium_install.sh new file mode 100755 index 0000000..b030785 --- /dev/null +++ b/src/foundation/scripts/cni/cilium/cilium_install.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -o xtrace +set -e + +umount /sys/fs/bpf || true +mount bpffs /sys/fs/bpf -t bpf diff --git a/src/foundation/scripts/cni/cilium/quick-install.yaml b/src/foundation/scripts/cni/cilium/quick-install.yaml new file mode 100644 index 0000000..1844b5d --- /dev/null +++ b/src/foundation/scripts/cni/cilium/quick-install.yaml @@ -0,0 +1,665 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation rule:line-length rule:comments rule:comments-indentation +--- +# Source: cilium/charts/config/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in a kvstore, etcd or consul, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + + # If you want to run cilium in debug mode change this value to true + debug: "true" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # ct-global-max-entries-* specifies the maximum number of connections + # supported across all endpoints, split by protocol: tcp or other. One pair + # of maps uses these values for IPv4 connections, and another pair of maps + # use these values for IPv6 connections. + # + # If these values are modified, then during the next Cilium startup the + # tracking of ongoing connections may be disrupted. This may lead to brief + # policy drops or a change in loadbalancing decisions for a connection. + # + # For users upgrading from Cilium 1.2 or earlier, to minimize disruption + # during the upgrade process, comment out these options. + bpf-ct-global-tcp-max: "524288" + bpf-ct-global-any-max: "262144" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # This may lead to policy drops or a change in loadbalancing decisions for a + # connection for some time. Endpoints may need to be recreated to restore + # connectivity. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: vxlan + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + + # DNS Polling periodically issues a DNS lookup for each `matchName` from + # cilium-agent. The result is used to regenerate endpoint policy. + # DNS lookups are repeated with an interval of 5 seconds, and are made for + # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP + # data is used instead. An IP change will trigger a regeneration of the Cilium + # policy for each endpoint and increment the per cilium-agent policy + # repository revision. + # + # This option is disabled by default starting from version 1.4.x in favor + # of a more powerful DNS proxy-based implementation, see [0] for details. + # Enable this option if you want to use FQDN policies but do not want to use + # the DNS proxy. + # + # To ease upgrade, users may opt to set this option to "true". + # Otherwise please refer to the Upgrade Guide [1] which explains how to + # prepare policy rules for upgrade. + # + # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based + # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action + tofqdns-enable-poller: "false" + + # wait-bpf-mount makes init container wait until bpf filesystem is mounted + wait-bpf-mount: "false" + + # Enable fetching of container-runtime specific metadata + # + # By default, the Kubernetes pod and namespace labels are retrieved and + # associated with endpoints for identification purposes. By integrating + # with the container runtime, container runtime specific labels can be + # retrieved, such labels will be prefixed with container: + # + # CAUTION: The container runtime labels can include information such as pod + # annotations which may result in each pod being associated a unique set of + # labels which can result in excessive security identities being allocated. + # Please review the labels filter when enabling container runtime labels. + # + # Supported values: + # - containerd + # - crio + # - docker + # - none + # - auto (automatically detect the container runtime) + # + container-runtime: none + + masquerade: "true" + + install-iptables-rules: "true" + auto-direct-node-routes: "false" + enable-node-port: "false" + +--- +# Source: cilium/charts/agent/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium + namespace: kube-system + +--- +# Source: cilium/charts/operator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium-operator + namespace: kube-system + +--- +# Source: cilium/charts/agent/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - nodes + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - patch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumendpoints + - ciliumendpoints/status + - ciliumnodes + - ciliumnodes/status + - ciliumidentities + - ciliumidentities/status + verbs: + - '*' + +--- +# Source: cilium/charts/operator/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + resources: + # to automatically read from k8s and import the node's pod CIDR to cilium's + # etcd so all nodes know how to reach another pod running in in a different + # node. + - nodes + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumendpoints + - ciliumendpoints/status + - ciliumnodes + - ciliumnodes/status + - ciliumidentities + - ciliumidentities/status + verbs: + - '*' + +--- +# Source: cilium/charts/agent/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: cilium + namespace: kube-system + +--- +# Source: cilium/charts/operator/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: cilium-operator + namespace: kube-system + +--- +# Source: cilium/charts/agent/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + name: cilium + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + template: + metadata: + annotations: + # This annotation plus the CriticalAddonsOnly toleration makes + # cilium to be a critical pod in the cluster, which ensures cilium + # gets priority scheduling. + # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + scheduler.alpha.kubernetes.io/critical-pod: "" + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]' + labels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + spec: + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_FLANNEL_MASTER_DEVICE + valueFrom: + configMapKeyRef: + key: flannel-master-device + name: cilium-config + optional: true + - name: CILIUM_FLANNEL_UNINSTALL_ON_EXIT + valueFrom: + configMapKeyRef: + key: flannel-uninstall-on-exit + name: cilium-config + optional: true + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + key: cni-chaining-mode + name: cilium-config + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + key: custom-cni-conf + name: cilium-config + optional: true + image: "iecedge/cilium:latest" + imagePullPolicy: IfNotPresent + lifecycle: + postStart: + exec: + command: + - /cni-install.sh + preStop: + exec: + command: + - /cni-uninstall.sh + livenessProbe: + exec: + command: + - cilium + - status + - --brief + failureThreshold: 10 + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + readinessProbe: + exec: + command: + - cilium + - status + - --brief + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + privileged: true + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/opt/cni/bin + name: cni-path + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + # Needed to be able to load kernel modules + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + hostNetwork: true + initContainers: + - command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + - name: CILIUM_WAIT_BPF_MOUNT + valueFrom: + configMapKeyRef: + key: wait-bpf-mount + name: cilium-config + optional: true + image: "iecedge/cilium:latest" + imagePullPolicy: IfNotPresent + name: clean-cilium-state + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: true + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + restartPolicy: Always + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + # To keep state between restarts / upgrades for bpf maps + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + # To install cilium cni plugin in the host + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + # To install cilium cni configuration in the host + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + # To be able to load kernel modules + - hostPath: + path: /lib/modules + name: lib-modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + defaultMode: 420 + optional: true + secretName: cilium-clustermesh + # To read the configuration from the config map + - configMap: + name: cilium-config + name: cilium-config-path + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + +--- +# Source: cilium/charts/operator/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + io.cilium/app: operator + name: cilium-operator + name: cilium-operator + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - args: + - --debug=$(CILIUM_DEBUG) + - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE) + command: + - cilium-operator + env: + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: CILIUM_CLUSTER_NAME + valueFrom: + configMapKeyRef: + key: cluster-name + name: cilium-config + optional: true + - name: CILIUM_CLUSTER_ID + valueFrom: + configMapKeyRef: + key: cluster-id + name: cilium-config + optional: true + - name: CILIUM_IPAM + valueFrom: + configMapKeyRef: + key: ipam + name: cilium-config + optional: true + - name: CILIUM_DISABLE_ENDPOINT_CRD + valueFrom: + configMapKeyRef: + key: disable-endpoint-crd + name: cilium-config + optional: true + - name: CILIUM_KVSTORE + valueFrom: + configMapKeyRef: + key: kvstore + name: cilium-config + optional: true + - name: CILIUM_KVSTORE_OPT + valueFrom: + configMapKeyRef: + key: kvstore-opt + name: cilium-config + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: cilium-aws + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: cilium-aws + optional: true + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + key: AWS_DEFAULT_REGION + name: cilium-aws + optional: true + - name: CILIUM_IDENTITY_ALLOCATION_MODE + valueFrom: + configMapKeyRef: + key: identity-allocation-mode + name: cilium-config + optional: true + image: "iecedge/operator:latest" + imagePullPolicy: IfNotPresent + name: cilium-operator + livenessProbe: + httpGet: + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + + hostNetwork: true + restartPolicy: Always + serviceAccount: cilium-operator + serviceAccountName: cilium-operator + +--- +# Source: cilium/charts/agent/templates/servicemonitor.yaml + +--- +# Source: cilium/charts/agent/templates/svc.yaml + +--- +# Source: cilium/charts/operator/templates/servicemonitor.yaml + +--- +# Source: cilium/charts/operator/templates/svc.yaml diff --git a/src/foundation/scripts/config b/src/foundation/scripts/config index 8c4837c..ebfcf6f 100755 --- a/src/foundation/scripts/config +++ b/src/foundation/scripts/config @@ -30,7 +30,7 @@ K8S_WORKER_GROUP=( CLUSTER_IP=172.16.1.136 # Align with the value in our K8s setup script POD_NETWORK_CIDR=192.168.0.0/16 SVC_CIDR=172.16.1.0/24 -#IEC support three kinds network solution for Kubernetes: calico,flannel,contivpp +#IEC support multiple network solution for Kubernetes: calico,flannel,contivpp,cilium,etc CNI_TYPE=calico #kubernetes-cni version 0.7.5/ 0.6.0 CNI_VERSION=0.6.0 diff --git a/src/foundation/scripts/setup-cni.sh b/src/foundation/scripts/setup-cni.sh index 76b963f..70b9767 100755 --- a/src/foundation/scripts/setup-cni.sh +++ b/src/foundation/scripts/setup-cni.sh @@ -97,6 +97,12 @@ install_danm(){ install_flannel } +install_cilium(){ + ${SCRIPTS_DIR}/cni/cilium/cilium_install.sh + + # Deploying cilium CNI + kubectl create -f ${SCRIPTS_DIR}/cni/cilium/quick-install.yaml +} case ${CNI_TYPE} in 'calico') echo "Install calico ..." @@ -122,6 +128,10 @@ case ${CNI_TYPE} in echo "Install danm ..." install_danm ;; + 'cilium') + echo "Install cilium ..." + install_cilium + ;; *) echo "${CNI_TYPE} is not supported" exit 1 diff --git a/src/foundation/scripts/startup.sh b/src/foundation/scripts/startup.sh index 40f54d3..4cc66f5 100755 --- a/src/foundation/scripts/startup.sh +++ b/src/foundation/scripts/startup.sh @@ -49,6 +49,10 @@ case ${CNI_TYPE} in DANM_CONFIG="cd iec/src/foundation/scripts/cni/danm && sudo ./danm_install.sh" sshpass -p ${passwd} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${ip_addr} $DANM_CONFIG ;; + cilium) + CILIUM_CONFIG="cd iec/src/foundation/scripts/cni/cilium && sudo ./cilium_install.sh" + sshpass -p ${passwd} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${ip_addr} $CILIUM_CONFIG + ;; *) ;; esac