From fa1c3405246cfa807b6c2e917d90ab8a44222bdb Mon Sep 17 00:00:00 2001 From: Trevor Tao Date: Mon, 20 Apr 2020 18:36:43 +0800 Subject: [PATCH] Upgrade ovn-kubernetes CNI to latest release Upgrade ovn-kubernetes CNI to latest release(the commit id: ef14b95bafd07c20cf253b165780c5e88241dbfd). This installation supports both arm64 and amd64 with multi-arched images. Fix an error in the install-ovn-k8s.sh script. Add a cleanup action before the installation to make sure there would be no obsolete ovs db information to bother the work of current ovs/ovn which is actually existed and bothered a lot. Signed-off-by: Trevor Tao Change-Id: Ia37f06d795a216979416d70c5fbd44765ce83837 Signed-off-by: Trevor Tao --- .../scripts/cni/ovn-kubernetes/clean_old_ovs.sh | 14 + .../scripts/cni/ovn-kubernetes/install-ovn-k8s.sh | 5 +- .../cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 | 122 +++++---- .../templates/ovnkube-db-raft.yaml.j2 | 297 ++++++++++++++++++++ .../templates/ovnkube-db-vip.yaml.j2 | 51 ++-- .../ovn-kubernetes/templates/ovnkube-db.yaml.j2 | 119 +++++--- .../templates/ovnkube-master.yaml.j2 | 150 ++++++----- .../templates/ovnkube-monitor.yaml.j2 | 124 +++++++++ .../ovn-kubernetes/templates/ovnkube-node.yaml.j2 | 127 +++++---- .../scripts/cni/ovn-kubernetes/yaml/ovn-setup.yaml | 137 ++++++++++ .../cni/ovn-kubernetes/yaml/ovnkube-db-raft.yaml | 298 +++++++++++++++++++++ .../cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml | 49 ++-- .../cni/ovn-kubernetes/yaml/ovnkube-db.yaml | 119 +++++--- .../cni/ovn-kubernetes/yaml/ovnkube-master.yaml | 154 ++++++----- .../cni/ovn-kubernetes/yaml/ovnkube-monitor.yaml | 125 +++++++++ .../cni/ovn-kubernetes/yaml/ovnkube-node.yaml | 123 +++++---- src/foundation/scripts/setup-cni.sh | 12 +- src/foundation/scripts/startup.sh | 4 + 18 files changed, 1644 insertions(+), 386 deletions(-) create mode 100755 src/foundation/scripts/cni/ovn-kubernetes/clean_old_ovs.sh create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-raft.yaml.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-monitor.yaml.j2 create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovn-setup.yaml create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-raft.yaml create mode 100644 src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-monitor.yaml diff --git a/src/foundation/scripts/cni/ovn-kubernetes/clean_old_ovs.sh b/src/foundation/scripts/cni/ovn-kubernetes/clean_old_ovs.sh new file mode 100755 index 0000000..0101da5 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/clean_old_ovs.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -o xtrace +set -e + +not_clean=${1:-} + +if [ -z ${not_clean} ] ; then + # Clean the old openvswitch db info + echo "Clean old ovs/ovn running dir ..." + sudo rm -rf /var/lib/openvswitch + sudo rm -rf /etc/ovn + sudo rm -rf /var/run/ovn-kubernetes + sudo rm -rf /etc/origin/openvswitch +fi diff --git a/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh b/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh index b858c96..4b0b622 100755 --- a/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh +++ b/src/foundation/scripts/cni/ovn-kubernetes/install-ovn-k8s.sh @@ -22,9 +22,12 @@ function wait_for { ) } +# shellcheck source=/dev/null +source ${SCRIPTS_DIR}/clean_old_ovs.sh + # Create OVN namespace, service accounts, ovnkube-db headless service, configmap, and policies kubectl create -f ${SCRIPTS_DIR}/yaml/ovn-setup.yaml -wait_for 5 'test $(kubectl get svc -n ovn-kubernetes | grep ovnkube-db -c ) -eq 1' +wait_for 5 'test $(kubectl get configmap -n ovn-kubernetes | grep ovn-config -c ) -eq 1' # Run ovnkube-db daemonset. diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 index c1d81d1..fd02efd 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 @@ -11,8 +11,6 @@ apiVersion: v1 kind: Namespace metadata: - annotations: - openshift.io/node-selector: "beta.kubernetes.io/os=linux" name: ovn-kubernetes --- @@ -33,64 +31,90 @@ metadata: name: ovn namespace: ovn-kubernetes +--- +# for now throw in all the privileges to run a pod. we can fine grain it further later. + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: ovn-kubernetes + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' + hostPID: true + hostIPC: true + hostNetwork: true + hostPorts: + - min: 0 + max: 65536 + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - name: system:ovn-reader + name: ovn-kubernetes rules: - apiGroups: - "" - - extensions resources: - pods - namespaces - - networkpolicies - nodes - verbs: - - get - - list - - watch + - endpoints + - services + - configmaps + verbs: ["get", "list", "watch"] - apiGroups: + - extensions - networking.k8s.io + - apps resources: - networkpolicies - verbs: - - get - - list - - watch + - statefulsets + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: - events - verbs: - - create - - patch - - update - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovn-reader -roleRef: - name: system:ovn-reader - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: -- kind: ServiceAccount - name: ovn - namespace: ovn-kubernetes + - endpoints + - configmaps + verbs: ["create", "patch", "update"] +- apiGroups: + - "" + resources: + - nodes + - pods + verbs: ["patch", "update"] +- apiGroups: + - extensions + - policy + resources: + - podsecuritypolicies + resourceNames: + - ovn-kubernetes + verbs: ["use"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: cluster-admin-0 + name: ovn-kubernetes roleRef: - name: cluster-admin + name: ovn-kubernetes kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: @@ -98,27 +122,6 @@ subjects: name: ovn namespace: ovn-kubernetes ---- -# service to expose the ovnkube-db pod -apiVersion: v1 -kind: Service -metadata: - name: ovnkube-db - namespace: ovn-kubernetes -spec: - ports: - - name: north - port: 6641 - protocol: TCP - targetPort: 6641 - - name: south - port: 6642 - protocol: TCP - targetPort: 6642 - sessionAffinity: None - clusterIP: None - type: ClusterIP - --- # The network cidr and service cidr are set in the ovn-config configmap kind: ConfigMap @@ -127,6 +130,7 @@ metadata: name: ovn-config namespace: ovn-kubernetes data: - net_cidr: "{{ net_cidr | default('10.128.0.0/14/23') }}" - svc_cidr: "{{ svc_cidr | default('172.30.0.0/16') }}" - k8s_apiserver: "{{ k8s_apiserver.stdout }}" + net_cidr: "{{ net_cidr }}" + svc_cidr: "{{ svc_cidr }}" + k8s_apiserver: "{{ k8s_apiserver }}" + mtu: "{{ mtu_value }}" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-raft.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-raft.yaml.j2 new file mode 100644 index 0000000..7642483 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-raft.yaml.j2 @@ -0,0 +1,297 @@ +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- + +# ovndb-raft PodDisruptBudget to prevent majority of ovnkube raft cluster +# nodes from disruption +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ovndb-raft-pdb + namespace: ovn-kubernetes +spec: + minAvailable: {{ ovn_db_minAvailable | default(2) }} + selector: + matchLabels: + name: ovnkube-db + +--- + +# ovnkube-db raft statefulset +# daemonset version 3 +# starts ovn NB/SB ovsdb daemons, each in a separate container +# +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ovnkube-db + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This statefulset launches the OVN Northbound/Southbound Database raft clusters. +spec: + serviceName: ovnkube-db + podManagementPolicy: "Parallel" + replicas: {{ ovn_db_replicas | default(3) }} + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-db + template: + metadata: + labels: + name: ovnkube-db + component: network + type: infra + kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + terminationGracePeriodSeconds: 30 + imagePullSecrets: + - name: registry-credentials + serviceAccountName: ovn + hostNetwork: true + + # required to be scheduled on node with k8s.ovn.org/ovnkube-db=true label but can + # only have one instance per node + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: k8s.ovn.org/ovnkube-db + operator: In + values: + - "true" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - ovnkube-db + topologyKey: kubernetes.io/hostname + + containers: + # nb-ovsdb - v3 + - name: nb-ovsdb + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "nb-ovsdb-raft"] + + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnnb-db-raft"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_NB + value: "{{ ovn_loglevel_nb }}" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + # end of container + + # sb-ovsdb - v3 + - name: sb-ovsdb + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "sb-ovsdb-raft"] + + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnsb-db-raft"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_SB + value: "{{ ovn_loglevel_sb }}" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + # end of container + + # db-metrics-exporter - v3 + - name: db-metrics-exporter + image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" + imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" + command: ["/root/ovnkube.sh", "db-raft-metrics"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + # end of container + + volumes: + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 index bea4a9a..556d9eb 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db-vip.yaml.j2 @@ -1,3 +1,25 @@ +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- + # ovnkube-db HA using Corosync/Pacemaker # daemonset version 3 # starts ovn NB/SB ovsdb daemons in a single container @@ -28,8 +50,7 @@ spec: name: ovnkube-db component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -70,13 +91,19 @@ spec: capabilities: add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) # and on the host in /var/lib/openvswitch/ - mountPath: /etc/openvswitch/ name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /etc/corosync name: host-etc-corosync - mountPath: /var/log/corosync @@ -91,8 +118,8 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_NB - value: "-vconsole:info -vfile:info" + - name: OVN_LOGLEVEL_NB + value: "{{ ovn_loglevel_nb }}" - name: K8S_APISERVER valueFrom: configMapKeyRef: @@ -102,20 +129,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: OVN_DB_VIP value: "{{ ovn_db_vip }}" - ports: - - name: healthz - containerPort: 10256 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10256 - # scheme: HTTP - lifecycle: # end of container volumes: diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 index 10523be..a810767 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-db.yaml.j2 @@ -1,7 +1,29 @@ +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- + # ovnkube-db # daemonset version 3 # starts ovn NB/SB ovsdb daemons, each in a separate container -# it is running on master node for now, but does not need to be the case +# it is running on master for now, but does not need to be the case kind: Deployment apiVersion: apps/v1 metadata: @@ -29,8 +51,7 @@ spec: name: ovnkube-db component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -55,13 +76,26 @@ spec: capabilities: add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) # and on the host in /var/lib/openvswitch/ - mountPath: /etc/openvswitch/ name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -70,8 +104,8 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_NB - value: "-vconsole:info -vfile:info" + - name: OVN_LOGLEVEL_NB + value: "{{ ovn_loglevel_nb }}" - name: K8S_APISERVER valueFrom: configMapKeyRef: @@ -81,18 +115,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10256 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10256 - # scheme: HTTP - lifecycle: + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnnb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 # end of container # sb-ovsdb - v3 @@ -107,13 +141,26 @@ spec: capabilities: add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) # and on the host in /var/lib/openvswitch/ - mountPath: /etc/openvswitch/ name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -122,8 +169,8 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_SB - value: "-vconsole:info -vfile:info" + - name: OVN_LOGLEVEL_SB + value: "{{ ovn_loglevel_sb }}" - name: K8S_APISERVER valueFrom: configMapKeyRef: @@ -133,23 +180,24 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10255 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10255 - # scheme: HTTP - lifecycle: + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnsb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of container nodeSelector: node-role.kubernetes.io/master: "" - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" volumes: - name: host-var-lib-ovs hostPath: @@ -157,5 +205,12 @@ spec: - name: host-var-log-ovs hostPath: path: /var/log/openvswitch + - name: host-slash + hostPath: + path: / + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate tolerations: - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 index b8bfc4f..f6a0b21 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-master.yaml.j2 @@ -1,7 +1,7 @@ # ovnkube-master # daemonset version 3 # starts master daemons, each in a separate container -# it is run on the master node(s) +# it is run on the master(s) kind: Deployment apiVersion: apps/v1 metadata: @@ -10,7 +10,7 @@ metadata: namespace: ovn-kubernetes annotations: kubernetes.io/description: | - This daemonset launches the ovn-kubernetes networking components. + This Deployment launches the ovn-kubernetes master networking components. spec: progressDeadlineSeconds: 600 replicas: 1 @@ -29,8 +29,7 @@ spec: name: ovnkube-master component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -39,10 +38,34 @@ spec: serviceAccountName: ovn hostNetwork: true - containers: + # required to be scheduled on a linux node with node-role.kubernetes.io/master label and + # only one instance of ovnkube-master pod per node + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + - key: kubernetes.io/os + operator: In + values: + - "linux" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - ovnkube-master + topologyKey: kubernetes.io/hostname - # run-ovn-northd - v3 - - name: run-ovn-northd + containers: + # ovn-northd - v3 + - name: ovn-northd image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" @@ -53,6 +76,7 @@ spec: capabilities: add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # Run directories where we need to be able to access sockets - mountPath: /var/run/dbus/ @@ -60,8 +84,15 @@ spec: readOnly: true - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -70,46 +101,28 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_NORTHD - value: "-vconsole:info" - - name: OVN_NET_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: net_cidr - - name: OVN_SVC_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: svc_cidr + - name: OVN_LOGLEVEL_NORTHD + value: "{{ ovn_loglevel_northd }}" - name: K8S_APISERVER valueFrom: configMapKeyRef: name: ovn-config key: k8s_apiserver - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: OVN_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10257 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10257 - # scheme: HTTP - lifecycle: + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-northd"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 # end of container - - name: run-nbctld + - name: nbctl-daemon image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" @@ -118,12 +131,19 @@ spec: securityContext: runAsUser: 0 + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /var/run/openvswitch/ name: host-var-run-ovs - + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: cpu: 100m @@ -131,24 +151,23 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" + - name: OVN_LOGLEVEL_NBCTLD + value: "{{ ovn_loglevel_nbctld }}" - name: K8S_APISERVER valueFrom: configMapKeyRef: name: ovn-config key: k8s_apiserver + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" - ports: - - name: healthz - containerPort: 10260 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10258 - # scheme: HTTP - lifecycle: + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-nbctld"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of container - name: ovnkube-master image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" @@ -159,6 +178,7 @@ spec: securityContext: runAsUser: 0 + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # Run directories where we need to be able to access sockets - mountPath: /var/run/dbus/ @@ -168,6 +188,11 @@ spec: name: host-var-log-ovnkube - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -177,7 +202,7 @@ spec: - name: OVN_DAEMONSET_VERSION value: "3" - name: OVNKUBE_LOGLEVEL - value: "4" + value: "{{ ovnkube_master_loglevel }}" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -201,23 +226,14 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10254 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10254 - # scheme: HTTP - lifecycle: + - name: OVN_HYBRID_OVERLAY_ENABLE + value: "{{ ovn_hybrid_overlay_enable }}" + - name: OVN_HYBRID_OVERLAY_NET_CIDR + value: "{{ ovn_hybrid_overlay_net_cidr }}" + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" # end of container - nodeSelector: - node-role.kubernetes.io/master: "" - beta.kubernetes.io/os: "linux" volumes: # TODO: Need to check why we need this? - name: host-var-run-dbus @@ -232,5 +248,9 @@ spec: - name: host-var-run-ovs hostPath: path: /var/run/openvswitch + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate tolerations: - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-monitor.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-monitor.yaml.j2 new file mode 100644 index 0000000..aa65fb6 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-monitor.yaml.j2 @@ -0,0 +1,124 @@ +# define ServiceMontior and Service resources for ovnkube-master, ovnkube-node, +# and ovnkube-db (required for prometheus monitoring) + +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ovnkube-master + name: monitor-ovnkube-master + namespace: ovn-kubernetes +spec: + endpoints: + - interval: 30s + port: http-metrics + scheme: http + path: /metrics + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes + selector: + matchLabels: + k8s-app: ovnkube-master +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-master + name: ovn-kubernetes-master-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-master + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9409 + protocol: TCP + targetPort: 9409 +--- + +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ovnkube-node + name: monitor-ovnkube-node + namespace: ovn-kubernetes +spec: + endpoints: + - interval: 30s + port: http-metrics + path: /metrics + scheme: http + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes + selector: + matchLabels: + k8s-app: ovnkube-node +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-node + name: ovn-kubernetes-node-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-node + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9410 + protocol: TCP + targetPort: 9410 + +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ovnkube-db + name: monitor-ovnkube-db + namespace: ovn-kubernetes +spec: + endpoints: + - interval: 30s + port: http-metrics + path: /metrics + scheme: http + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes + selector: + matchLabels: + k8s-app: ovnkube-db +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-db + name: ovn-kubernetes-db-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-db + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9476 + protocol: TCP + targetPort: 9476 diff --git a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 index 8e7b181..713e426 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 +++ b/src/foundation/scripts/cni/ovn-kubernetes/templates/ovnkube-node.yaml.j2 @@ -11,7 +11,7 @@ metadata: namespace: ovn-kubernetes annotations: kubernetes.io/description: | - This daemonset launches the ovn-kubernetes networking components. + This DaemonSet launches the ovn-kubernetes networking components for worker nodes. spec: selector: matchLabels: @@ -22,10 +22,10 @@ spec: metadata: labels: app: ovnkube-node + name: ovnkube-node component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -48,14 +48,22 @@ spec: command: - /usr/share/openvswitch/scripts/ovs-ctl - status - initialDelaySeconds: 15 - periodSeconds: 5 + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovs-daemons"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 securityContext: runAsUser: 0 # Permission could be reduced by selecting an appropriate SELinux policy privileged: true + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /lib/modules name: host-modules @@ -100,14 +108,22 @@ spec: capabilities: add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/run/dbus/ name: host-var-run-dbus readOnly: true - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -116,44 +132,26 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVNKUBE_LOGLEVEL - value: "4" - - name: OVN_NET_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: net_cidr - - name: OVN_SVC_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: svc_cidr + - name: OVN_LOG_CONTROLLER + value: "{{ ovn_loglevel_controller }}" - name: K8S_APISERVER valueFrom: configMapKeyRef: name: ovn-config key: k8s_apiserver - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: OVN_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" - ports: - - name: healthz - containerPort: 10258 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10258 - # scheme: HTTP - lifecycle: + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-controller"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 - name: ovnkube-node image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" @@ -165,8 +163,16 @@ spec: runAsUser: 0 capabilities: add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"] + {% if kind is defined and kind -%} + privileged: true + {% endif %} + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true - mountPath: /var/run/dbus/ name: host-var-run-dbus readOnly: true @@ -174,6 +180,8 @@ spec: name: host-var-log-ovnkube - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs # We mount our socket here - mountPath: /var/run/ovn-kubernetes name: host-var-run-ovn-kubernetes @@ -182,6 +190,14 @@ spec: name: host-opt-cni-bin - mountPath: /etc/cni/net.d name: host-etc-cni-netd + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + {% if kind is defined and kind -%} + - mountPath: /var/run/netns + name: host-netns + mountPropagation: Bidirectional + {% endif %} resources: requests: @@ -191,7 +207,7 @@ spec: - name: OVN_DAEMONSET_VERSION value: "3" - name: OVNKUBE_LOGLEVEL - value: "5" + value: "{{ ovnkube_node_loglevel }}" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -207,6 +223,11 @@ spec: configMapKeyRef: name: ovn-config key: k8s_apiserver + - name: OVN_MTU + valueFrom: + configMapKeyRef: + name: ovn-config + key: mtu - name: K8S_NODE valueFrom: fieldRef: @@ -215,25 +236,26 @@ spec: value: "{{ ovn_gateway_mode }}" - name: OVN_GATEWAY_OPTS value: "{{ ovn_gateway_opts }}" + - name: OVN_HYBRID_OVERLAY_ENABLE + value: "{{ ovn_hybrid_overlay_enable }}" + - name: OVN_HYBRID_OVERLAY_NET_CIDR + value: "{{ ovn_hybrid_overlay_net_cidr }}" + - name: OVN_SSL_ENABLE + value: "{{ ovn_ssl_en }}" - ports: - - name: healthz - containerPort: 10259 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10259 - # scheme: HTTP lifecycle: preStop: exec: command: ["/root/ovnkube.sh", "cleanup-ovn-node"] + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnkube-node"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: @@ -266,8 +288,21 @@ spec: - name: host-etc-cni-netd hostPath: path: /etc/cni/net.d + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate + - name: host-slash + hostPath: + path: / - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch + {% if kind is defined and kind -%} + - name: host-netns + hostPath: + path: /var/run/netns + {% endif %} + tolerations: - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovn-setup.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovn-setup.yaml new file mode 100644 index 0000000..28c2dfb --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovn-setup.yaml @@ -0,0 +1,137 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +--- +# ovn-namespace.yaml +# +# Setup for Kubernetes to support the ovn-kubernetes plugin +# +# Create the namespace for ovn-kubernetes. +# +# This provisioning is done as part of installation after the cluster is +# up and before the ovn daemonsets are created. + +apiVersion: v1 +kind: Namespace +metadata: + name: ovn-kubernetes + +--- +# ovn-policy.yaml +# +# Setup for Kubernetes to support the ovn-kubernetes plugin +# +# Create the service account and policies. +# ovnkube interacts with kubernetes and the environment +# must be properly set up. +# +# This provisioning is done as part of installation after the cluster is +# up and before the ovn daemonsets are created. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovn + namespace: ovn-kubernetes + +--- +# for now throw in all the privileges to run a pod. we can fine grain it further later. + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: ovn-kubernetes + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' + hostPID: true + hostIPC: true + hostNetwork: true + hostPorts: + - min: 0 + max: 65536 + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ovn-kubernetes +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + - nodes + - endpoints + - services + - configmaps + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + - networking.k8s.io + - apps + resources: + - networkpolicies + - statefulsets + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - events + - endpoints + - configmaps + verbs: ["create", "patch", "update"] +- apiGroups: + - "" + resources: + - nodes + - pods + verbs: ["patch", "update"] +- apiGroups: + - extensions + - policy + resources: + - podsecuritypolicies + resourceNames: + - ovn-kubernetes + verbs: ["use"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-kubernetes +roleRef: + name: ovn-kubernetes + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: ovn + namespace: ovn-kubernetes + +--- +# The network cidr and service cidr are set in the ovn-config configmap +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovn-config + namespace: ovn-kubernetes +data: + net_cidr: "192.168.0.0/16" + svc_cidr: "172.16.1.0/24" + k8s_apiserver: "https://10.169.41.225:6443" + mtu: "1400" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-raft.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-raft.yaml new file mode 100644 index 0000000..e42f4d7 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-raft.yaml @@ -0,0 +1,298 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- + +# ovndb-raft PodDisruptBudget to prevent majority of ovnkube raft cluster +# nodes from disruption +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ovndb-raft-pdb + namespace: ovn-kubernetes +spec: + minAvailable: 2 + selector: + matchLabels: + name: ovnkube-db + +--- + +# ovnkube-db raft statefulset +# daemonset version 3 +# starts ovn NB/SB ovsdb daemons, each in a separate container +# +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ovnkube-db + namespace: ovn-kubernetes + annotations: + kubernetes.io/description: | + This statefulset launches the OVN Northbound/Southbound Database raft clusters. +spec: + serviceName: ovnkube-db + podManagementPolicy: "Parallel" + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ovnkube-db + template: + metadata: + labels: + name: ovnkube-db + component: network + type: infra + kubernetes.io/os: "linux" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + terminationGracePeriodSeconds: 30 + imagePullSecrets: + - name: registry-credentials + serviceAccountName: ovn + hostNetwork: true + + # required to be scheduled on node with k8s.ovn.org/ovnkube-db=true label but can + # only have one instance per node + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: k8s.ovn.org/ovnkube-db + operator: In + values: + - "true" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - ovnkube-db + topologyKey: kubernetes.io/hostname + + containers: + # nb-ovsdb - v3 + - name: nb-ovsdb + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" + imagePullPolicy: "IfNotPresent" + command: ["/root/ovnkube.sh", "nb-ovsdb-raft"] + + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnnb-db-raft"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_NB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "no" + # end of container + + # sb-ovsdb - v3 + - name: sb-ovsdb + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" + imagePullPolicy: "IfNotPresent" + command: ["/root/ovnkube.sh", "sb-ovsdb-raft"] + + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnsb-db-raft"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_SB + value: "-vconsole:info -vfile:info" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "no" + # end of container + + # db-metrics-exporter - v3 + - name: db-metrics-exporter + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" + imagePullPolicy: "IfNotPresent" + command: ["/root/ovnkube.sh", "db-raft-metrics"] + + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) + # and on the host in /var/lib/openvswitch/ + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_SSL_ENABLE + value: "no" + # end of container + + volumes: + - name: host-var-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-run-ovs + hostPath: + path: /var/run/openvswitch + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate + tolerations: + - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml index 672581e..757477a 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db-vip.yaml @@ -1,4 +1,26 @@ # yamllint disable rule:hyphens rule:commas rule:indentation +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- + # ovnkube-db HA using Corosync/Pacemaker # daemonset version 3 # starts ovn NB/SB ovsdb daemons in a single container @@ -29,8 +51,7 @@ spec: name: ovnkube-db component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -71,13 +92,19 @@ spec: capabilities: add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) # and on the host in /var/lib/openvswitch/ - mountPath: /etc/openvswitch/ name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /etc/corosync name: host-etc-corosync - mountPath: /var/log/corosync @@ -92,7 +119,7 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_NB + - name: OVN_LOGLEVEL_NB value: "-vconsole:info -vfile:info" - name: K8S_APISERVER valueFrom: @@ -103,20 +130,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: OVN_DB_VIP value: "" - ports: - - name: healthz - containerPort: 10256 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10256 - # scheme: HTTP - lifecycle: # end of container volumes: diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml index 602e27b..bf81495 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-db.yaml @@ -1,8 +1,30 @@ # yamllint disable rule:hyphens rule:commas rule:indentation +# service to expose the ovnkube-db pod +apiVersion: v1 +kind: Service +metadata: + name: ovnkube-db + namespace: ovn-kubernetes +spec: + ports: + - name: north + port: 6641 + protocol: TCP + targetPort: 6641 + - name: south + port: 6642 + protocol: TCP + targetPort: 6642 + sessionAffinity: None + clusterIP: None + type: ClusterIP + +--- + # ovnkube-db # daemonset version 3 # starts ovn NB/SB ovsdb daemons, each in a separate container -# it is running on master node for now, but does not need to be the case +# it is running on master for now, but does not need to be the case kind: Deployment apiVersion: apps/v1 metadata: @@ -30,8 +52,7 @@ spec: name: ovnkube-db component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -46,7 +67,7 @@ spec: # nb-ovsdb - v3 - name: nb-ovsdb - image: "iecedge/ovn-daemonset:latest" + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "nb-ovsdb"] @@ -56,13 +77,26 @@ spec: capabilities: add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) # and on the host in /var/lib/openvswitch/ - mountPath: /etc/openvswitch/ name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -71,7 +105,7 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_NB + - name: OVN_LOGLEVEL_NB value: "-vconsole:info -vfile:info" - name: K8S_APISERVER valueFrom: @@ -82,23 +116,23 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10256 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10256 - # scheme: HTTP - lifecycle: + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "no" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnnb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 # end of container # sb-ovsdb - v3 - name: sb-ovsdb - image: "iecedge/ovn-daemonset:latest" + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "sb-ovsdb"] @@ -108,13 +142,26 @@ spec: capabilities: add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # ovn db is stored in the pod in /etc/openvswitch + # (or in /etc/ovn if OVN from new repository is used) # and on the host in /var/lib/openvswitch/ - mountPath: /etc/openvswitch/ name: host-var-lib-ovs + - mountPath: /etc/ovn/ + name: host-var-lib-ovs - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -123,7 +170,7 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_SB + - name: OVN_LOGLEVEL_SB value: "-vconsole:info -vfile:info" - name: K8S_APISERVER valueFrom: @@ -134,23 +181,24 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10255 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10255 - # scheme: HTTP - lifecycle: + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OVN_SSL_ENABLE + value: "no" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnsb-db"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of container nodeSelector: node-role.kubernetes.io/master: "" - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" volumes: - name: host-var-lib-ovs hostPath: @@ -158,5 +206,12 @@ spec: - name: host-var-log-ovs hostPath: path: /var/log/openvswitch + - name: host-slash + hostPath: + path: / + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate tolerations: - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml index de271b5..952f3c4 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-master.yaml @@ -2,7 +2,7 @@ # ovnkube-master # daemonset version 3 # starts master daemons, each in a separate container -# it is run on the master node(s) +# it is run on the master(s) kind: Deployment apiVersion: apps/v1 metadata: @@ -11,7 +11,7 @@ metadata: namespace: ovn-kubernetes annotations: kubernetes.io/description: | - This daemonset launches the ovn-kubernetes networking components. + This Deployment launches the ovn-kubernetes master networking components. spec: progressDeadlineSeconds: 600 replicas: 1 @@ -30,8 +30,7 @@ spec: name: ovnkube-master component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -40,11 +39,35 @@ spec: serviceAccountName: ovn hostNetwork: true - containers: + # required to be scheduled on a linux node with node-role.kubernetes.io/master label and + # only one instance of ovnkube-master pod per node + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + - key: kubernetes.io/os + operator: In + values: + - "linux" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - ovnkube-master + topologyKey: kubernetes.io/hostname - # run-ovn-northd - v3 - - name: run-ovn-northd - image: "iecedge/ovn-daemonset:latest" + containers: + # ovn-northd - v3 + - name: ovn-northd + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "run-ovn-northd"] @@ -54,6 +77,7 @@ spec: capabilities: add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # Run directories where we need to be able to access sockets - mountPath: /var/run/dbus/ @@ -61,8 +85,15 @@ spec: readOnly: true - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -71,47 +102,29 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVN_LOG_NORTHD - value: "-vconsole:info" - - name: OVN_NET_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: net_cidr - - name: OVN_SVC_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: svc_cidr + - name: OVN_LOGLEVEL_NORTHD + value: "-vconsole:info -vfile:info" - name: K8S_APISERVER valueFrom: configMapKeyRef: name: ovn-config key: k8s_apiserver - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: OVN_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10257 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10257 - # scheme: HTTP - lifecycle: + - name: OVN_SSL_ENABLE + value: "no" + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-northd"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 # end of container - - name: run-nbctld - image: "iecedge/ovn-daemonset:latest" + - name: nbctl-daemon + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "run-nbctld"] @@ -119,12 +132,19 @@ spec: securityContext: runAsUser: 0 + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /var/run/openvswitch/ name: host-var-run-ovs - + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: cpu: 100m @@ -132,27 +152,26 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" + - name: OVN_LOGLEVEL_NBCTLD + value: "-vconsole:info" - name: K8S_APISERVER valueFrom: configMapKeyRef: name: ovn-config key: k8s_apiserver + - name: OVN_SSL_ENABLE + value: "no" - ports: - - name: healthz - containerPort: 10260 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10258 - # scheme: HTTP - lifecycle: + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-nbctld"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # end of container - name: ovnkube-master - image: "iecedge/ovn-daemonset:latest" + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "ovn-master"] @@ -160,6 +179,7 @@ spec: securityContext: runAsUser: 0 + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # Run directories where we need to be able to access sockets - mountPath: /var/run/dbus/ @@ -169,6 +189,11 @@ spec: name: host-var-log-ovnkube - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -202,23 +227,14 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - ports: - - name: healthz - containerPort: 10254 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10254 - # scheme: HTTP - lifecycle: + - name: OVN_HYBRID_OVERLAY_ENABLE + value: "" + - name: OVN_HYBRID_OVERLAY_NET_CIDR + value: "" + - name: OVN_SSL_ENABLE + value: "no" # end of container - nodeSelector: - node-role.kubernetes.io/master: "" - beta.kubernetes.io/os: "linux" volumes: # TODO: Need to check why we need this? - name: host-var-run-dbus @@ -233,5 +249,9 @@ spec: - name: host-var-run-ovs hostPath: path: /var/run/openvswitch + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate tolerations: - operator: "Exists" diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-monitor.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-monitor.yaml new file mode 100644 index 0000000..fe537f8 --- /dev/null +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-monitor.yaml @@ -0,0 +1,125 @@ +# yamllint disable rule:hyphens rule:commas rule:indentation +# define ServiceMontior and Service resources for ovnkube-master, ovnkube-node, +# and ovnkube-db (required for prometheus monitoring) + +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ovnkube-master + name: monitor-ovnkube-master + namespace: ovn-kubernetes +spec: + endpoints: + - interval: 30s + port: http-metrics + scheme: http + path: /metrics + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes + selector: + matchLabels: + k8s-app: ovnkube-master +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-master + name: ovn-kubernetes-master-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-master + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9409 + protocol: TCP + targetPort: 9409 +--- + +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ovnkube-node + name: monitor-ovnkube-node + namespace: ovn-kubernetes +spec: + endpoints: + - interval: 30s + port: http-metrics + path: /metrics + scheme: http + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes + selector: + matchLabels: + k8s-app: ovnkube-node +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-node + name: ovn-kubernetes-node-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-node + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9410 + protocol: TCP + targetPort: 9410 + +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ovnkube-db + name: monitor-ovnkube-db + namespace: ovn-kubernetes +spec: + endpoints: + - interval: 30s + port: http-metrics + path: /metrics + scheme: http + jobLabel: k8s-app + namespaceSelector: + matchNames: + - ovn-kubernetes + selector: + matchLabels: + k8s-app: ovnkube-db +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: ovnkube-db + name: ovn-kubernetes-db-prometheus-discovery + namespace: ovn-kubernetes +spec: + selector: + name: ovnkube-db + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 9476 + protocol: TCP + targetPort: 9476 diff --git a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml index 3c08485..a371538 100644 --- a/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml +++ b/src/foundation/scripts/cni/ovn-kubernetes/yaml/ovnkube-node.yaml @@ -12,7 +12,7 @@ metadata: namespace: ovn-kubernetes annotations: kubernetes.io/description: | - This daemonset launches the ovn-kubernetes networking components. + This DaemonSet launches the ovn-kubernetes networking components for worker nodes. spec: selector: matchLabels: @@ -23,10 +23,10 @@ spec: metadata: labels: app: ovnkube-node + name: ovnkube-node component: network type: infra - openshift.io/component: network - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: @@ -39,7 +39,7 @@ spec: # ovsdb-server and ovs-switchd daemons - name: ovs-daemons - image: "iecedge/ovn-daemonset:latest" + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "ovs-server"] @@ -49,14 +49,22 @@ spec: command: - /usr/share/openvswitch/scripts/ovs-ctl - status - initialDelaySeconds: 15 - periodSeconds: 5 + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovs-daemons"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 securityContext: runAsUser: 0 # Permission could be reduced by selecting an appropriate SELinux policy privileged: true + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /lib/modules name: host-modules @@ -91,7 +99,7 @@ spec: command: ["/root/ovnkube.sh", "cleanup-ovs-server"] - name: ovn-controller - image: "iecedge/ovn-daemonset:latest" + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "ovn-controller"] @@ -101,14 +109,22 @@ spec: capabilities: add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/run/dbus/ name: host-var-run-dbus readOnly: true - mountPath: /var/log/openvswitch/ name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true resources: requests: @@ -117,47 +133,29 @@ spec: env: - name: OVN_DAEMONSET_VERSION value: "3" - - name: OVNKUBE_LOGLEVEL - value: "4" - - name: OVN_NET_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: net_cidr - - name: OVN_SVC_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: svc_cidr + - name: OVN_LOG_CONTROLLER + value: "-vconsole:info" - name: K8S_APISERVER valueFrom: configMapKeyRef: name: ovn-config key: k8s_apiserver - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: OVN_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OVN_SSL_ENABLE + value: "no" - ports: - - name: healthz - containerPort: 10258 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10258 - # scheme: HTTP - lifecycle: + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-controller"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 - name: ovnkube-node - image: "iecedge/ovn-daemonset:latest" + image: "iecedge/ovn-daemonset-ubuntu:2020-04-16" imagePullPolicy: "IfNotPresent" command: ["/root/ovnkube.sh", "ovn-node"] @@ -167,7 +165,13 @@ spec: capabilities: add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"] + + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true - mountPath: /var/run/dbus/ name: host-var-run-dbus readOnly: true @@ -175,6 +179,8 @@ spec: name: host-var-log-ovnkube - mountPath: /var/run/openvswitch/ name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs # We mount our socket here - mountPath: /var/run/ovn-kubernetes name: host-var-run-ovn-kubernetes @@ -183,6 +189,10 @@ spec: name: host-opt-cni-bin - mountPath: /etc/cni/net.d name: host-etc-cni-netd + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + resources: requests: @@ -192,7 +202,7 @@ spec: - name: OVN_DAEMONSET_VERSION value: "3" - name: OVNKUBE_LOGLEVEL - value: "5" + value: "4" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -208,6 +218,11 @@ spec: configMapKeyRef: name: ovn-config key: k8s_apiserver + - name: OVN_MTU + valueFrom: + configMapKeyRef: + name: ovn-config + key: mtu - name: K8S_NODE valueFrom: fieldRef: @@ -216,25 +231,26 @@ spec: value: "local" - name: OVN_GATEWAY_OPTS value: "" + - name: OVN_HYBRID_OVERLAY_ENABLE + value: "" + - name: OVN_HYBRID_OVERLAY_NET_CIDR + value: "" + - name: OVN_SSL_ENABLE + value: "no" - ports: - - name: healthz - containerPort: 10259 - # TODO: Temporarily disabled until we determine how to wait for clean default - # config - # livenessProbe: - # initialDelaySeconds: 10 - # httpGet: - # path: /healthz - # port: 10259 - # scheme: HTTP lifecycle: preStop: exec: command: ["/root/ovnkube.sh", "cleanup-ovn-node"] + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnkube-node"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: @@ -267,8 +283,17 @@ spec: - name: host-etc-cni-netd hostPath: path: /etc/cni/net.d + - name: host-ovn-cert + hostPath: + path: /etc/ovn + type: DirectoryOrCreate + - name: host-slash + hostPath: + path: / - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch + + tolerations: - operator: "Exists" diff --git a/src/foundation/scripts/setup-cni.sh b/src/foundation/scripts/setup-cni.sh index f0689de..7eb8eb2 100755 --- a/src/foundation/scripts/setup-cni.sh +++ b/src/foundation/scripts/setup-cni.sh @@ -52,21 +52,25 @@ install_contivpp(){ install_ovn_kubernetes(){ # Update the ovn-kubernetes yaml files - net_cidr_repl="{{ net_cidr | default('10.128.0.0/14/23') }}" - svc_cidr_repl="{{ svc_cidr | default('172.30.0.0/16') }}" - k8s_apiserver_repl="{{ k8s_apiserver.stdout }}" + net_cidr_repl="{{ net_cidr }}" + svc_cidr_repl="{{ svc_cidr }}" + k8s_apiserver_repl="{{ k8s_apiserver }}" + mtu_repl="{{ mtu_value }}" k8s_apiserver="https://${K8S_MASTER_IP}:6443" net_cidr="${POD_NETWORK_CIDR}" svc_cidr="${SERVICE_CIDR}" + mtu_def_value=1400 echo "net_cidr: ${net_cidr}" echo "svc_cidr: ${svc_cidr}" echo "k8s_apiserver: ${k8s_apiserver}" + echo "mtu: ${mtu_def_value}" sed "s,${net_cidr_repl},${net_cidr}, s,${svc_cidr_repl},${svc_cidr}, - s,${k8s_apiserver_repl},${k8s_apiserver}," \ + s,${k8s_apiserver_repl},${k8s_apiserver}, + s,${mtu_repl},${mtu_def_value}," \ ${SCRIPTS_DIR}/cni/ovn-kubernetes/templates/ovn-setup.yaml.j2 > \ ${SCRIPTS_DIR}/cni/ovn-kubernetes/yaml/ovn-setup.yaml diff --git a/src/foundation/scripts/startup.sh b/src/foundation/scripts/startup.sh index 1687d9d..3b8525b 100755 --- a/src/foundation/scripts/startup.sh +++ b/src/foundation/scripts/startup.sh @@ -54,6 +54,10 @@ case ${CNI_TYPE} in CILIUM_CONFIG="cd iec/src/foundation/scripts/cni/cilium && sudo ./cilium_install.sh" sshpass -p ${passwd} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${ip_addr} $CILIUM_CONFIG ;; + ovn-kubernetes) + OVN_KUBERNETES_PRECONFIG="cd iec/src/foundation/scripts/cni/ovn-kubernetes && ./clean_old_ovs.sh" + sshpass -p ${passwd} ssh -o StrictHostKeyChecking=no ${HOST_USER}@${ip_addr} $OVN_KUBERNETES_PRECONFIG + ;; *) ;; esac -- 2.16.6