From a4546182269b01038a1e672cb16b081930bd11bb Mon Sep 17 00:00:00 2001 From: Trevor Tao Date: Thu, 24 Oct 2019 11:11:32 +0800 Subject: [PATCH] Add OVN-Kubernetes CNI Installation for IEC Add OVN-Kubernetes CNI installation scripts for IEC: 1. the install-ovn-k8s.sh for installation by yamls 2. Needed yaml files and yaml template files 3. Update setup-cni.sh to add a new CNI type of ovn-kubernetes 4. Update config and startup.sh to add more parameters used in the installation. 5. Add fat-manifest pushing scripts 6. Fixing the deficiency by adding more parameters when calling k8s_master.sh in startup.sh Signed-off-by: Trevor Tao Change-Id: I534edae5baa79b7729654c28de66ee076ed62359 Signed-off-by: Trevor Tao --- .../scripts/cni/ovn-kubernetes/install-ovn-k8s.sh | 46 ++++ .../scripts/cni/ovn-kubernetes/push-manifest.sh | 36 +++ .../templates/cleanup-ovn-cni.conf.j2 | 2 + .../cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 | 132 ++++++++++ .../templates/ovnkube-db-vip.yaml.j2 | 139 +++++++++++ .../ovn-kubernetes/templates/ovnkube-db.yaml.j2 | 161 ++++++++++++ .../templates/ovnkube-master.yaml.j2 | 236 ++++++++++++++++++ .../ovn-kubernetes/templates/ovnkube-node.yaml.j2 | 273 ++++++++++++++++++++ .../cni/ovn-kubernetes/uninstall-ovn-k8s.sh | 23 ++ .../cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml | 140 +++++++++++ .../cni/ovn-kubernetes/yaml/ovnkube-db.yaml | 162 ++++++++++++ .../cni/ovn-kubernetes/yaml/ovnkube-master.yaml | 237 ++++++++++++++++++ .../cni/ovn-kubernetes/yaml/ovnkube-node.yaml | 274 +++++++++++++++++++++ src/foundation/scripts/config | 1 + src/foundation/scripts/setup-cni.sh | 36 ++- src/foundation/scripts/startup.sh | 4 +- 16 files changed, 1899 insertions(+), 3 deletions(-) create mode 100755 src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh create mode 100755 src/foundation/scripts/cni/ovn-kubernetes/push-manifest.sh create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/cleanup-ovn-cni.conf.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 create mode 100755 src/foundation/scripts/cni/ovn-kubernetes/uninstall-ovn-k8s.sh create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml diff --git a/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh b/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh new file mode 100755 index 0000000..b858c96 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh @@ -0,0 +1,46 @@ +#!/bin/bash -ex +# shellcheck disable=SC2016 + +SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") + +function wait_for { + # Execute in a subshell to prevent local variable override during recursion + ( + local total_attempts=$1; shift + local cmdstr=$* + local sleep_time=2 + echo -e "\n[wait_for] Waiting for cmd to return success: ${cmdstr}" + # shellcheck disable=SC2034 + for attempt in $(seq "${total_attempts}"); do + echo "[wait_for] Attempt ${attempt}/${total_attempts%.*} for: ${cmdstr}" + # shellcheck disable=SC2015 + eval "${cmdstr}" && echo "[wait_for] OK: ${cmdstr}" && return 0 || true + sleep "${sleep_time}" + done + echo "[wait_for] ERROR: Failed after max attempts: ${cmdstr}" + return 1 + ) +} + +# Create OVN namespace, service accounts, ovnkube-db headless service, configmap, and policies +kubectl create -f ${SCRIPTS_DIR}/yaml/ovn-setup.yaml +wait_for 5 'test $(kubectl get svc -n ovn-kubernetes | grep ovnkube-db -c ) -eq 1' + + +# Run ovnkube-db daemonset. +kubectl create -f ${SCRIPTS_DIR}/yaml/ovnkube-db.yaml +wait_for 60 'test $(kubectl get pods -n ovn-kubernetes | grep -e "ovnkube-db" | grep "Running" -c) -eq 1' + + +# Run ovnkube-master daemonset. +kubectl create -f ${SCRIPTS_DIR}/yaml/ovnkube-master.yaml +wait_for 60 'test $(kubectl get pods -n ovn-kubernetes | grep -e "ovnkube-master" | grep "Running" -c) -eq 1' + + +# Run ovnkube daemonsets for nodes, maybe more than 1 ovnkube-node pods since there would be 1 ovnkube-node +# pod on each K8s node +kubectl create -f ${SCRIPTS_DIR}/yaml/ovnkube-node.yaml +wait_for 60 'test $(kubectl get pods -n ovn-kubernetes | grep -e "ovnkube-node" | grep "Running" -c) -ge 1' + + +#kubectl get pods -n ovn-kubernetes diff --git a/src/foundation/scripts/cni/ovn-kubernetes/push-manifest.sh b/src/foundation/scripts/cni/ovn-kubernetes/push-manifest.sh new file mode 100755 index 0000000..f4e3c80 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/push-manifest.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +#Supported platforms of multi-arch images are: amd64 arm64 +LINUX_ARCH=(amd64 arm64) +PLATFORMS=linux/${LINUX_ARCH[0]} +for i in $(seq 1 $[${#LINUX_ARCH[@]}-1]) +do + PLATFORMS=$PLATFORMS,linux/${LINUX_ARCH[$i]} +done + +IMAGES_OVN=("ovn-daemonset") +#IMAGES_OVN=("ovn-daemonset" "ovn-daemonset-u") +BRANCH_TAG=latest + +#Before push, 'docker login' is needed +push_multi_arch(){ + + if [ ! -f "./manifest-tool" ] + then + sudo apt-get install -y jq + wget https://github.com/estesp/manifest-tool/releases/download/v0.9.0/manifest-tool-linux-${BUILDARCH} \ + -O manifest-tool && \ + chmod +x ./manifest-tool + fi + + for IMAGE in "${IMAGES_OVN[@]}" + do + echo "multi arch image: ""iecedge/${IMAGE}" + ./manifest-tool push from-args --platforms ${PLATFORMS} --template iecedge/${IMAGE}-ARCH:${BRANCH_TAG} \ + --target iecedge/${IMAGE}:${BRANCH_TAG} + done +} + +echo "Push fat manifest for multi-arch images:" +push_multi_arch + diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/cleanup-ovn-cni.conf.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/cleanup-ovn-cni.conf.j2 new file mode 100644 index 0000000..a184074 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/cleanup-ovn-cni.conf.j2 @@ -0,0 +1,2 @@ +r /etc/cni/net.d/10-ovn-kubernetes.conf +r /etc/origin/openvswitch/conf.db diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 new file mode 100644 index 0000000..c1d81d1 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 @@ -0,0 +1,132 @@ +--- +# ovn-namespace.yaml +# +# Setup for Kubernetes to support the ovn-kubernetes plugin +# +# Create the namespace for ovn-kubernetes. +# +# This provisioning is done as part of installation after the cluster is +# up and before the ovn daemonsets are created. + +apiVersion: v1 +kind: Namespace +metadata: + annotations: + openshift.io/node-selector: "beta.kubernetes.io/os=linux" + name: ovn-kubernetes + +--- +# ovn-policy.yaml +# +# Setup for Kubernetes to support the ovn-kubernetes plugin +# +# Create the service account and policies. +# ovnkube interacts with kubernetes and the environment +# must be properly set up. +# +# This provisioning is done as part of installation after the cluster is +# up and before the ovn daemonsets are created. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovn + namespace: ovn-kubernetes + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:ovn-reader +rules: +- apiGroups: + - "" + - extensions + resources: + - pods + - namespaces + - networkpolicies + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-reader +roleRef: + name: system:ovn-reader + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: ovn + namespace: ovn-kubernetes + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-admin-0 +roleRef: + name: cluster-admin + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: ovn + namespace: ovn-kubernetes + +--- +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- +# The network cidr and service cidr are set in the ovn-config configmap +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovn-config + namespace: ovn-kubernetes +data: + net_cidr: "{{ net_cidr | default('10.128.0.0/14/23') }}" + svc_cidr: "{{ svc_cidr | default('172.30.0.0/16') }}" + k8s_apiserver: "{{ k8s_apiserver.stdout }}" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 new file mode 100644 index 0000000..bea4a9a --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 @@ -0,0 +1,139 @@ +# ovnkube-db HA using Corosync/Pacemaker +# daemonset version 3 +# starts ovn NB/SB ovsdb daemons in a single container +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovnkube-db + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the OVN NB/SB DB server in a single container. +spec: + progressDeadlineSeconds: 600 + replicas: {{ ovn_db_replicas | default(3) }} + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-db + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + name: ovnkube-db + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + # required to be scheduled on node with openvswitch.org/ovnkube-db=true label but can + # only have one instance per node + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: openvswitch.org/ovnkube-db + operator: In + values: + - "true" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - ovnkube-db + topologyKey: kubernetes.io/hostname + + containers: + # ovsdb with corosync in one container - v3 + - name: ovsdb + image: "{{ ovn_db_vip_image | default('docker.io/ovnkube/ovndb-vip-u:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovndb-vip.sh", "run-ovndb"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /etc/corosync + name: host-etc-corosync + - mountPath: /var/log/corosync + name: host-var-log-corosync + - mountPath: /dev/shm + name: dshm + + resources: + requests: + cpu: 100m + memory: 1024Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_NB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_VIP + value: "{{ ovn_db_vip }}" + ports: + - name: healthz + containerPort: 10256 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10256 + # scheme: HTTP + lifecycle: + # end of container + + volumes: + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-log-corosync + hostPath: + path: /var/log/corosync + - name: host-etc-corosync + hostPath: + path: /etc/corosync + - name: dshm + emptyDir: + medium: Memory + + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 new file mode 100644 index 0000000..10523be --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 @@ -0,0 +1,161 @@ +# ovnkube-db +# daemonset version 3 +# starts ovn NB/SB ovsdb daemons, each in a separate container +# it is running on master node for now, but does not need to be the case +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovnkube-db + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the OVN NB/SB ovsdb service components. +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-db + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + name: ovnkube-db + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + containers: + # firewall rules for ovn - assumed to be setup + # iptables -A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 6641 -j ACCEPT + # iptables -A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 6642 -j ACCEPT + + # nb-ovsdb - v3 + - name: nb-ovsdb + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "nb-ovsdb"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_NB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10256 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10256 + # scheme: HTTP + lifecycle: + # end of container + + # sb-ovsdb - v3 + - name: sb-ovsdb + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "sb-ovsdb"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_SB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10255 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10255 + # scheme: HTTP + lifecycle: + # end of container + + nodeSelector: + node-role.kubernetes.io/master: "" + beta.kubernetes.io/os: "linux" + volumes: + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 new file mode 100644 index 0000000..b8bfc4f --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 @@ -0,0 +1,236 @@ +# ovnkube-master +# daemonset version 3 +# starts master daemons, each in a separate container +# it is run on the master node(s) +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovnkube-master + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the ovn-kubernetes networking components. +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-master + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + name: ovnkube-master + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + + containers: + + # run-ovn-northd - v3 + - name: run-ovn-northd + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "run-ovn-northd"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + + volumeMounts: + # Run directories where we need to be able to access sockets + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_NORTHD + value: "-vconsole:info" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10257 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10257 + # scheme: HTTP + lifecycle: + # end of container + + - name: run-nbctld + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "run-nbctld"] + + securityContext: + runAsUser: 0 + + volumeMounts: + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + + ports: + - name: healthz + containerPort: 10260 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10258 + # scheme: HTTP + lifecycle: + + - name: ovnkube-master + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "ovn-master"] + + securityContext: + runAsUser: 0 + + volumeMounts: + # Run directories where we need to be able to access sockets + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/ovn-kubernetes/ + name: host-var-log-ovnkube + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVNKUBE_LOGLEVEL + value: "4" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10254 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10254 + # scheme: HTTP + lifecycle: + # end of container + + nodeSelector: + node-role.kubernetes.io/master: "" + beta.kubernetes.io/os: "linux" + volumes: + # TODO: Need to check why we need this? + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-log-ovnkube + hostPath: + path: /var/log/ovn-kubernetes + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 new file mode 100644 index 0000000..8e7b181 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 @@ -0,0 +1,273 @@ +--- +# ovnkube-node +# daemonset version 3 +# starts node daemons for ovs and ovn, each in a separate container +# it is run on all nodes +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovnkube-node + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the ovn-kubernetes networking components. +spec: + selector: + matchLabels: + app: ovnkube-node + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: ovnkube-node + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + hostPID: true + containers: + + # ovsdb-server and ovs-switchd daemons + - name: ovs-daemons + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "ovs-server"] + + livenessProbe: + exec: + command: + - /usr/share/openvswitch/scripts/ovs-ctl + - status + initialDelaySeconds: 15 + periodSeconds: 5 + + securityContext: + runAsUser: 0 + # Permission could be reduced by selecting an appropriate SELinux policy + privileged: true + + volumeMounts: + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /run/openvswitch + name: host-run-ovs + - mountPath: /var/run/openvswitch + name: host-var-run-ovs + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 200m + memory: 400Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + lifecycle: + preStop: + exec: + command: ["/root/ovnkube.sh", "cleanup-ovs-server"] + + - name: ovn-controller + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "ovn-controller"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVNKUBE_LOGLEVEL + value: "4" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + ports: + - name: healthz + containerPort: 10258 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10258 + # scheme: HTTP + lifecycle: + + - name: ovnkube-node + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + + command: ["/root/ovnkube.sh", "ovn-node"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"] + + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/ovn-kubernetes/ + name: host-var-log-ovnkube + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + # We mount our socket here + - mountPath: /var/run/ovn-kubernetes + name: host-var-run-ovn-kubernetes + # CNI related mounts which we take over + - mountPath: /opt/cni/bin + name: host-opt-cni-bin + - mountPath: /etc/cni/net.d + name: host-etc-cni-netd + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVNKUBE_LOGLEVEL + value: "5" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_GATEWAY_MODE + value: "{{ ovn_gateway_mode }}" + - name: OVN_GATEWAY_OPTS + value: "{{ ovn_gateway_opts }}" + + ports: + - name: healthz + containerPort: 10259 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10259 + # scheme: HTTP + lifecycle: + preStop: + exec: + command: ["/root/ovnkube.sh", "cleanup-ovn-node"] + + nodeSelector: + beta.kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-log-ovnkube + hostPath: + path: /var/log/ovn-kubernetes + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + - name: host-var-run-ovn-kubernetes + hostPath: + path: /var/run/ovn-kubernetes + - name: host-sys + hostPath: + path: /sys + - name: host-opt-cni-bin + hostPath: + path: /opt/cni/bin + - name: host-etc-cni-netd + hostPath: + path: /etc/cni/net.d + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/uninstall-ovn-k8s.sh b/src/foundation/scripts/cni/ovn-kubernetes/uninstall-ovn-k8s.sh new file mode 100755 index 0000000..ea3834a --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/uninstall-ovn-k8s.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex +# shellcheck disable=SC1073,SC1072,SC1039,SC2059 + +SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Run ovnkube daemonsets for nodes +kubectl delete -f ${SCRIPTS_DIR}/yaml/ovnkube-node.yaml +sleep 3 + +# Run ovnkube-master daemonset. +kubectl delete -f ${SCRIPTS_DIR}/yaml/ovnkube-master.yaml +sleep 3 + + +# Delete ovnkube-db daemonset. +kubectl delete -f ${SCRIPTS_DIR}/yaml/ovnkube-db.yaml +sleep 3 + +# Delete OVN namespace, service accounts, ovnkube-db headless service, configmap, and policies +kubectl delete -f ${SCRIPTS_DIR}/yaml/ovn-setup.yaml +sleep 2 + +#kubectl get pods -n ovn-kubernetes diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml new file mode 100644 index 0000000..672581e --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml @@ -0,0 +1,140 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +# ovnkube-db HA using Corosync/Pacemaker +# daemonset version 3 +# starts ovn NB/SB ovsdb daemons in a single container +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovnkube-db + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the OVN NB/SB DB server in a single container. +spec: + progressDeadlineSeconds: 600 + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-db + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + name: ovnkube-db + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + # required to be scheduled on node with openvswitch.org/ovnkube-db=true label but can + # only have one instance per node + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: openvswitch.org/ovnkube-db + operator: In + values: + - "true" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - ovnkube-db + topologyKey: kubernetes.io/hostname + + containers: + # ovsdb with corosync in one container - v3 + - name: ovsdb + image: "docker.io/ovnkube/ovndb-vip-u:latest" + imagePullPolicy: "IfNotPresent" + command: ["/root/ovndb-vip.sh", "run-ovndb"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /etc/corosync + name: host-etc-corosync + - mountPath: /var/log/corosync + name: host-var-log-corosync + - mountPath: /dev/shm + name: dshm + + resources: + requests: + cpu: 100m + memory: 1024Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_NB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_VIP + value: "" + ports: + - name: healthz + containerPort: 10256 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10256 + # scheme: HTTP + lifecycle: + # end of container + + volumes: + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-log-corosync + hostPath: + path: /var/log/corosync + - name: host-etc-corosync + hostPath: + path: /etc/corosync + - name: dshm + emptyDir: + medium: Memory + + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml new file mode 100644 index 0000000..602e27b --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml @@ -0,0 +1,162 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +# ovnkube-db +# daemonset version 3 +# starts ovn NB/SB ovsdb daemons, each in a separate container +# it is running on master node for now, but does not need to be the case +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovnkube-db + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the OVN NB/SB ovsdb service components. +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-db + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + name: ovnkube-db + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + containers: + # firewall rules for ovn - assumed to be setup + # iptables -A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 6641 -j ACCEPT + # iptables -A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 6642 -j ACCEPT + + # nb-ovsdb - v3 + - name: nb-ovsdb + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "nb-ovsdb"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_NB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10256 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10256 + # scheme: HTTP + lifecycle: + # end of container + + # sb-ovsdb - v3 + - name: sb-ovsdb + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "sb-ovsdb"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_SB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10255 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10255 + # scheme: HTTP + lifecycle: + # end of container + + nodeSelector: + node-role.kubernetes.io/master: "" + beta.kubernetes.io/os: "linux" + volumes: + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml new file mode 100644 index 0000000..de271b5 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml @@ -0,0 +1,237 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +# ovnkube-master +# daemonset version 3 +# starts master daemons, each in a separate container +# it is run on the master node(s) +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovnkube-master + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the ovn-kubernetes networking components. +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-master + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + name: ovnkube-master + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + + containers: + + # run-ovn-northd - v3 + - name: run-ovn-northd + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "run-ovn-northd"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + + volumeMounts: + # Run directories where we need to be able to access sockets + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOG_NORTHD + value: "-vconsole:info" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10257 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10257 + # scheme: HTTP + lifecycle: + # end of container + + - name: run-nbctld + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "run-nbctld"] + + securityContext: + runAsUser: 0 + + volumeMounts: + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + + ports: + - name: healthz + containerPort: 10260 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10258 + # scheme: HTTP + lifecycle: + + - name: ovnkube-master + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "ovn-master"] + + securityContext: + runAsUser: 0 + + volumeMounts: + # Run directories where we need to be able to access sockets + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/ovn-kubernetes/ + name: host-var-log-ovnkube + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVNKUBE_LOGLEVEL + value: "4" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 10254 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10254 + # scheme: HTTP + lifecycle: + # end of container + + nodeSelector: + node-role.kubernetes.io/master: "" + beta.kubernetes.io/os: "linux" + volumes: + # TODO: Need to check why we need this? + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-log-ovnkube + hostPath: + path: /var/log/ovn-kubernetes + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml new file mode 100644 index 0000000..3c08485 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml @@ -0,0 +1,274 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +--- +# ovnkube-node +# daemonset version 3 +# starts node daemons for ovs and ovn, each in a separate container +# it is run on all nodes +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovnkube-node + # namespace set up by install + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the ovn-kubernetes networking components. +spec: + selector: + matchLabels: + app: ovnkube-node + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: ovnkube-node + component: network + type: infra + openshift.io/component: network + beta.kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + # Requires fairly broad permissions - ability to read all services and network functions as well + # as all pods. + serviceAccountName: ovn + hostNetwork: true + hostPID: true + containers: + + # ovsdb-server and ovs-switchd daemons + - name: ovs-daemons + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "ovs-server"] + + livenessProbe: + exec: + command: + - /usr/share/openvswitch/scripts/ovs-ctl + - status + initialDelaySeconds: 15 + periodSeconds: 5 + + securityContext: + runAsUser: 0 + # Permission could be reduced by selecting an appropriate SELinux policy + privileged: true + + volumeMounts: + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /run/openvswitch + name: host-run-ovs + - mountPath: /var/run/openvswitch + name: host-var-run-ovs + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 200m + memory: 400Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + lifecycle: + preStop: + exec: + command: ["/root/ovnkube.sh", "cleanup-ovs-server"] + + - name: ovn-controller + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "ovn-controller"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVNKUBE_LOGLEVEL + value: "4" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + ports: + - name: healthz + containerPort: 10258 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10258 + # scheme: HTTP + lifecycle: + + - name: ovnkube-node + image: "iecedge/ovn-daemonset:latest" + imagePullPolicy: "IfNotPresent" + + command: ["/root/ovnkube.sh", "ovn-node"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"] + + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/ovn-kubernetes/ + name: host-var-log-ovnkube + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + # We mount our socket here + - mountPath: /var/run/ovn-kubernetes + name: host-var-run-ovn-kubernetes + # CNI related mounts which we take over + - mountPath: /opt/cni/bin + name: host-opt-cni-bin + - mountPath: /etc/cni/net.d + name: host-etc-cni-netd + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVNKUBE_LOGLEVEL + value: "5" + - name: OVN_NET_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: net_cidr + - name: OVN_SVC_CIDR + valueFrom: + configMapKeyRef: + name: ovn-config + key: svc_cidr + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_GATEWAY_MODE + value: "local" + - name: OVN_GATEWAY_OPTS + value: "" + + ports: + - name: healthz + containerPort: 10259 + # TODO: Temporarily disabled until we determine how to wait for clean default + # config + # livenessProbe: + # initialDelaySeconds: 10 + # httpGet: + # path: /healthz + # port: 10259 + # scheme: HTTP + lifecycle: + preStop: + exec: + command: ["/root/ovnkube.sh", "cleanup-ovn-node"] + + nodeSelector: + beta.kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-log-ovnkube + hostPath: + path: /var/log/ovn-kubernetes + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + - name: host-var-run-ovn-kubernetes + hostPath: + path: /var/run/ovn-kubernetes + - name: host-sys + hostPath: + path: /sys + - name: host-opt-cni-bin + hostPath: + path: /opt/cni/bin + - name: host-etc-cni-netd + hostPath: + path: /etc/cni/net.d + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/config b/src/foundation/scripts/config index f4c27b0..8c4837c 100755 --- a/src/foundation/scripts/config +++ b/src/foundation/scripts/config @@ -29,6 +29,7 @@ K8S_WORKER_GROUP=( # K8s parameter CLUSTER_IP=172.16.1.136 # Align with the value in our K8s setup script POD_NETWORK_CIDR=192.168.0.0/16 +SVC_CIDR=172.16.1.0/24 #IEC support three kinds network solution for Kubernetes: calico,flannel,contivpp CNI_TYPE=calico #kubernetes-cni version 0.7.5/ 0.6.0 diff --git a/src/foundation/scripts/setup-cni.sh b/src/foundation/scripts/setup-cni.sh index d1b27bc..fb02ff3 100755 --- a/src/foundation/scripts/setup-cni.sh +++ b/src/foundation/scripts/setup-cni.sh @@ -11,7 +11,9 @@ fi CLUSTER_IP=${1:-172.16.1.136} # Align with the value in our K8s setup script POD_NETWORK_CIDR=${2:-192.168.0.0/16} CNI_TYPE=${3:-calico} -DEV_NAME=${4:-} +K8S_MASTER_IP=${4:-10.169.41.173} +SERVICE_CIDR=${5:-172.16.1.0/24} +DEV_NAME=${6:-} SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") @@ -46,6 +48,34 @@ install_contivpp(){ kubectl apply -f "${SCRIPTS_DIR}/cni/contivpp/contiv-vpp.yaml" } +install_ovn_kubernetes(){ + # Update the ovn-kubernetes yaml files + + net_cidr_repl="{{ net_cidr | default('10.128.0.0/14/23') }}" + svc_cidr_repl="{{ svc_cidr | default('172.30.0.0/16') }}" + k8s_apiserver_repl="{{ k8s_apiserver.stdout }}" + + k8s_apiserver="https://${K8S_MASTER_IP}:6443" + net_cidr="${POD_NETWORK_CIDR}" + svc_cidr="${SERVICE_CIDR}" + + echo "net_cidr: ${net_cidr}" + echo "svc_cidr: ${svc_cidr}" + echo "k8s_apiserver: ${k8s_apiserver}" + + sed "s,${net_cidr_repl},${net_cidr}, + s,${svc_cidr_repl},${svc_cidr}, + s,${k8s_apiserver_repl},${k8s_apiserver}," \ + ${SCRIPTS_DIR}/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 > \ + ${SCRIPTS_DIR}/cni/ovn-kubernetes/yaml/ovn-setup.yaml + + # Install ovn-kubernetes by yaml files + # shellcheck source=/dev/null + source ${SCRIPTS_DIR}/cni/ovn-kubernetes/install-ovn-k8s.sh + +} + + case ${CNI_TYPE} in 'calico') echo "Install calico ..." @@ -59,6 +89,10 @@ case ${CNI_TYPE} in echo "Install Contiv-VPP ..." install_contivpp ;; + 'ovn-kubernetes') + echo "Install Ovn-Kubernetes ..." + install_ovn_kubernetes + ;; *) echo "${CNI_TYPE} is not supported" exit 1 diff --git a/src/foundation/scripts/startup.sh b/src/foundation/scripts/startup.sh index f9f137d..2bdc95f 100755 --- a/src/foundation/scripts/startup.sh +++ b/src/foundation/scripts/startup.sh @@ -47,7 +47,7 @@ deploy_k8s () { cd iec/src/foundation/scripts/ && source k8s_common.sh $KUBE_VERSION $CNI_VERSION" #Automatic deploy the K8s environments on Master node - SETUP_MASTER="cd iec/src/foundation/scripts/ && source k8s_master.sh ${K8S_MASTER_IP}" + SETUP_MASTER="cd iec/src/foundation/scripts/ && source k8s_master.sh ${K8S_MASTER_IP} ${POD_NETWORK_CIDR} ${SVC_CIDR}" sshpass -p ${K8S_MASTERPW} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${K8S_MASTER_IP} ${INSTALL_SOFTWARE} sshpass -p ${K8S_MASTERPW} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${K8S_MASTER_IP} ${SETUP_MASTER} | tee ${LOG_FILE} @@ -83,7 +83,7 @@ deploy_k8s () { #Deploy etcd & CNI from master node - SETUP_CNI="cd iec/src/foundation/scripts && source setup-cni.sh $CLUSTER_IP $POD_NETWORK_CIDR $CNI_TYPE" + SETUP_CNI="cd iec/src/foundation/scripts && source setup-cni.sh $CLUSTER_IP $POD_NETWORK_CIDR $CNI_TYPE $K8S_MASTER_IP $SVC_CIDR" sshpass -p ${K8S_MASTERPW} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${K8S_MASTER_IP} ${SETUP_CNI} SETUP_HELM="cd iec/src/foundation/scripts && source helm.sh" sshpass -p ${K8S_MASTERPW} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${K8S_MASTER_IP} ${SETUP_HELM} -- 2.16.6