From df4ab230a9f2e86297236807a39e50194f9a345e Mon Sep 17 00:00:00 2001 From: trevor tao Date: Mon, 31 Oct 2022 21:55:55 +0800 Subject: [PATCH] Upgrade Calico to v3.24.3 Signed-off-by: trevor tao Change-Id: Ia3369e9136033542eee5fafc06fb636520682fcb --- ...vxlan.yaml => calico-3.24.3-install-vxlan.yaml} | 289 +++++++++++++-------- src/foundation/scripts/setup-cni.sh | 2 +- 2 files changed, 186 insertions(+), 105 deletions(-) rename src/foundation/scripts/cni/calico/k8s-new/{calico-3.23-install-vxlan.yaml => calico-3.24.3-install-vxlan.yaml} (97%) diff --git a/src/foundation/scripts/cni/calico/k8s-new/calico-3.23-install-vxlan.yaml b/src/foundation/scripts/cni/calico/k8s-new/calico-3.24.3-install-vxlan.yaml similarity index 97% rename from src/foundation/scripts/cni/calico/k8s-new/calico-3.23-install-vxlan.yaml rename to src/foundation/scripts/cni/calico/k8s-new/calico-3.24.3-install-vxlan.yaml index 42adb41..54ab835 100644 --- a/src/foundation/scripts/cni/calico/k8s-new/calico-3.23-install-vxlan.yaml +++ b/src/foundation/scripts/cni/calico/k8s-new/calico-3.24.3-install-vxlan.yaml @@ -1,4 +1,34 @@ --- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -52,10 +82,8 @@ data: } ] } - --- # Source: calico/templates/kdd-crds.yaml - apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -67,6 +95,7 @@ spec: listKind: BGPConfigurationList plural: bgpconfigurations singular: bgpconfiguration + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -234,8 +263,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -247,6 +276,7 @@ spec: listKind: BGPPeerList plural: bgppeers singular: bgppeer + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -353,8 +383,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -366,6 +396,7 @@ spec: listKind: BlockAffinityList plural: blockaffinities singular: blockaffinity + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -414,8 +445,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -430,6 +461,7 @@ spec: listKind: CalicoNodeStatusList plural: caliconodestatuses singular: caliconodestatus + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -677,8 +709,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -690,6 +722,7 @@ spec: listKind: ClusterInformationList plural: clusterinformations singular: clusterinformation + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -741,8 +774,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -754,6 +787,7 @@ spec: listKind: FelixConfigurationList plural: felixconfigurations singular: felixconfiguration + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -831,7 +865,7 @@ spec: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection - are routed within the host and how is routing intepreted by RPF + are routed within the host and how is routing interpreted by RPF check. [Default: 0]' type: integer bpfExternalServiceMode: @@ -844,6 +878,11 @@ spec: node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' type: string + bpfHostConntrackBypass: + description: 'BPFHostConntrackBypass Controls whether to bypass Linux + conntrack in BPF mode for workloads and services. [Default: true + - bypass Linux conntrack]' + type: boolean bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls whether Felix's embedded kube-proxy accepts EndpointSlices or not. @@ -879,6 +918,11 @@ spec: policy. Selectors such as "all()" can result in large numbers of entries (one entry per endpoint in that case). type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer bpfMapSizeNATAffinity: type: integer bpfMapSizeNATBackend: @@ -911,6 +955,11 @@ spec: are inclusive. [Default: 20000:29999]' pattern: ^.* x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1040,7 +1089,6 @@ spec: are auto-detected. type: string floatingIPs: - default: Disabled description: FloatingIPs configures whether or not Felix will program floating IP addresses. enum: @@ -1302,6 +1350,10 @@ spec: information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean routeTableRange: description: Deprecated in favor of RouteTableRanges. Calico programs additional Linux route tables for various purposes. RouteTableRange @@ -1363,8 +1415,8 @@ spec: type: boolean vxlanEnabled: description: 'VXLANEnabled overrides whether Felix should create the - VXLAN tunnel device for VXLAN networking. Optional as Felix determines - this based on the existing IP pools. [Default: nil (unset)]' + VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix + determines this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel @@ -1379,7 +1431,13 @@ spec: vxlanVNI: type: integer wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled. + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). [Default: false]' type: boolean wireguardHostEncryptionEnabled: @@ -1388,7 +1446,11 @@ spec: type: boolean wireguardInterfaceName: description: 'WireguardInterfaceName specifies the name to use for - the Wireguard interface. [Default: wg.calico]' + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive @@ -1396,11 +1458,19 @@ spec: type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used - by Wireguard. [Default: 51820]' + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' type: integer wireguardMTU: - description: 'WireguardMTU controls the MTU on the Wireguard interface. - See Configuring MTU [Default: 1420]' + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' type: integer wireguardRoutingRulePriority: description: 'WireguardRoutingRulePriority controls the priority value @@ -1432,8 +1502,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -1445,6 +1515,7 @@ spec: listKind: GlobalNetworkPolicyList plural: globalnetworkpolicies singular: globalnetworkpolicy + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2287,8 +2358,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2300,6 +2371,7 @@ spec: listKind: GlobalNetworkSetList plural: globalnetworksets singular: globalnetworkset + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2340,8 +2412,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2353,6 +2425,7 @@ spec: listKind: HostEndpointList plural: hostendpoints singular: hostendpoint + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2448,8 +2521,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2461,6 +2534,7 @@ spec: listKind: IPAMBlockList plural: ipamblocks singular: ipamblock + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2567,8 +2641,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2580,6 +2654,7 @@ spec: listKind: IPAMConfigList plural: ipamconfigs singular: ipamconfig + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2607,6 +2682,8 @@ spec: maxBlocksPerHost: description: MaxBlocksPerHost, if non-zero, is the max number of blocks that can be affine to each host. + maximum: 2147483647 + minimum: 0 type: integer strictAffinity: type: boolean @@ -2623,8 +2700,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2636,6 +2713,7 @@ spec: listKind: IPAMHandleList plural: ipamhandles singular: ipamhandle + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2679,8 +2757,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2692,6 +2770,7 @@ spec: listKind: IPPoolList plural: ippools singular: ippool + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2763,7 +2842,7 @@ spec: for internal use only.' type: boolean natOutgoing: - description: When nat-outgoing is true, packets sent from Calico networked + description: When natOutgoing is true, packets sent from Calico networked containers in this pool to destinations outside of this pool will be masqueraded. type: boolean @@ -2788,8 +2867,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2804,6 +2883,7 @@ spec: listKind: IPReservationList plural: ipreservations singular: ipreservation + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -2842,8 +2922,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2855,6 +2935,7 @@ spec: listKind: KubeControllersConfigurationList plural: kubecontrollersconfigurations singular: kubecontrollersconfiguration + preserveUnknownFields: false scope: Cluster versions: - name: v1 @@ -3095,8 +3176,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -3108,6 +3189,7 @@ spec: listKind: NetworkPolicyList plural: networkpolicies singular: networkpolicy + preserveUnknownFields: false scope: Namespaced versions: - name: v1 @@ -3931,8 +4013,8 @@ status: plural: "" conditions: [] storedVersions: [] - --- +# Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -3944,6 +4026,7 @@ spec: listKind: NetworkSetList plural: networksets singular: networkset + preserveUnknownFields: false scope: Namespaced versions: - name: v1 @@ -3982,11 +4065,8 @@ status: plural: "" conditions: [] storedVersions: [] - ---- --- # Source: calico/templates/calico-kube-controllers-rbac.yaml - # Include a clusterrole for the kube-controllers component, # and bind it to the calico-kube-controllers serviceaccount. kind: ClusterRole @@ -4068,21 +4148,6 @@ rules: - update # watch for changes - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- - --- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, @@ -4092,6 +4157,14 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-node + verbs: + - create # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] resources: @@ -4221,11 +4294,14 @@ rules: - create - update - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration - apiGroups: ["crd.projectcalico.org"] resources: - ipamconfigs verbs: - get + - create # Block affinities must also be watchable by confd for route aggregation. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4239,8 +4315,22 @@ rules: - daemonsets verbs: - get - --- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -4253,7 +4343,6 @@ subjects: - kind: ServiceAccount name: calico-node namespace: kube-system - --- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well @@ -4301,7 +4390,8 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.23.1 + image: docker.io/calico/cni:v3.24.3 + imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: @@ -4328,7 +4418,8 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.23.1 + image: docker.io/calico/cni:v3.24.3 + imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: - configMapRef: @@ -4366,12 +4457,38 @@ spec: name: cni-net-dir securityContext: privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: docker.io/calico/node:v3.24.3 + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.23.1 + image: docker.io/calico/node:v3.24.3 + imagePullPolicy: IfNotPresent envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. @@ -4433,8 +4550,8 @@ spec: # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. - - name: CALICO_IPV4POOL_CIDR - value: "192.168.0.0/16" + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -4496,11 +4613,8 @@ spec: mountPath: /var/run/nodeagent # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the # parent directory. - - name: sysfs - mountPath: /sys/fs/ - # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. - # If the host is known to mount that filesystem already then Bidirectional can be omitted. - mountPropagation: Bidirectional + - name: bpffs + mountPath: /sys/fs/bpf - name: cni-log-dir mountPath: /var/log/calico/cni readOnly: true @@ -4519,10 +4633,18 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate - - name: sysfs + - name: sys-fs hostPath: path: /sys/fs/ type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc # Used to install CNI. - name: cni-bin-dir hostPath: @@ -4546,14 +4668,6 @@ spec: type: DirectoryOrCreate path: /var/run/nodeagent --- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers apiVersion: apps/v1 @@ -4586,11 +4700,14 @@ spec: operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule serviceAccountName: calico-kube-controllers priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.23.1 + image: docker.io/calico/kube-controllers:v3.24.3 + imagePullPolicy: IfNotPresent env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS @@ -4612,39 +4729,3 @@ spec: - /usr/bin/check-status - -r periodSeconds: 10 - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system - ---- - -# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - ---- -# Source: calico/templates/calico-etcd-secrets.yaml - ---- -# Source: calico/templates/calico-typha.yaml - ---- -# Source: calico/templates/configure-canal.yaml - - diff --git a/src/foundation/scripts/setup-cni.sh b/src/foundation/scripts/setup-cni.sh index 75fa1f2..835ec84 100755 --- a/src/foundation/scripts/setup-cni.sh +++ b/src/foundation/scripts/setup-cni.sh @@ -27,7 +27,7 @@ install_calico(){ if [[ $kube_version > "v1.15.0" ]]; then sed -i "s@192.168.0.0/16@${POD_NETWORK_CIDR}@" \ "${SCRIPTS_DIR}/cni/calico/k8s-new/calico-multi-arch.yaml" - kubectl create -f "${SCRIPTS_DIR}/cni/calico/k8s-new/calico-3.23-install-vxlan.yaml" + kubectl create -f "${SCRIPTS_DIR}/cni/calico/k8s-new/calico-3.24.3-install-vxlan.yaml" else # Install the Etcd Database ETCD_YAML=etcd.yaml -- 2.16.6