src/foundation/scripts/cni/multus/multus-sriov-calico/multus-sriov-calico-daemonsets-k8s-v1.16.yaml

   1 # yamllint disable
   2 # This yaml file contains necessary configuration to setup
   3 # a demo environment for Multus + SR-IOV, the config includes
   4 # the following pieces:
   5 # 1. Multus ConfigMap
   6 # 2. Network Plumbing Working Group Spec Version 1 CustomerResourceDefinition
   7 # 3. Multus ClusterRole & ClusterRoleBinding
   8 # 4. Multus & SR-IOV Device Plugin ServiceAccounts
   9 # 5. Multus & SR-IOV Device Plugin & SR-IOV CNI DaemonSets
  10
  11 # Note: This yaml file will not create customer SR-IOV CRD
  12 # which will be specified in Pod spec annotation. Below is
  13 # an example of SR-IOV CRD:
  14 #
  15 # apiVersion: "k8s.cni.cncf.io/v1"
  16 # kind: NetworkAttachmentDefinition
  17 # metadata:
  18 #   name: sriov-net1
  19 #   annotations:
  20 #     k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
  21 # spec:
  22 #   config: '{
  23 #        "type": "sriov",
  24 #        "name": "sriov-network",
  25 #        "ipam": {
  26 #                "type": "host-local",
  27 #                "subnet": "10.56.217.0/24",
  28 #                "routes": [{
  29 #                        "dst": "0.0.0.0/0"
  30 #                }],
  31 #                "gateway": "10.56.217.1"
  32 #        }
  33 #   }'
  34
  35 # An example of Pod spec using above SR-IOV CRD:
  36 #
  37 # apiVersion: v1
  38 # kind: Pod
  39 # metadata:
  40 #   name: testpod1
  41 #   labels:
  42 #     env: test
  43 #   annotations:
  44 #     k8s.v1.cni.cncf.io/networks: sriov-net1
  45 # spec:
  46 #   containers:
  47 #   - name: appcntr1
  48 #     image: centos/tools
  49 #     imagePullPolicy: IfNotPresent
  50 #     command: [ "/bin/bash", "-c", "--" ]
  51 #     args: [ "while true; do sleep 300000; done;" ]
  52 #     resources:
  53 #       requests:
  54 #         intel.com/sriov: '1'
  55 #       limits:
  56 #        intel.com/sriov: '1'
  57
  58
  59 # --------------------------------------------------------------------
  60
  61 # 1. Multus ConfigMap
  62 #
  63 # This configMap assumes that:
  64 # - Kubeconfig file is located at "/etc/kubernetes/admin.conf" on host
  65 # - Default master plugin for Multus is set to flannel
  66 #
  67 # Note: If either of above is not True in your environment
  68 # make sure they are properly set to the corrent values.
  69 ---
  70 kind: ConfigMap
  71 apiVersion: v1
  72 metadata:
  73   name: multus-cni-config
  74   namespace: kube-system
  75   labels:
  76     tier: node
  77     app: multus
  78 data:
  79   cni-conf.json: |
  80     {
  81       "name": "multus-cni-network",
  82       "type": "multus",
  83       "capabilities": {
  84         "portMappings": true
  85       },
  86       "delegates": [
  87         {
  88           "cniVersion": "0.3.1",
  89           "name": "default-cni-network",
  90           "plugins": [
  91             {
  92               "name": "k8s-pod-network",
  93               "cniVersion": "0.3.0",
  94               "type": "calico",
  95               "log_level": "info",
  96               "datastore_type": "kubernetes",
  97               "nodename": "__KUBERNETES_NODE_NAME__",
  98               "mtu": 1440,
  99               "ipam": {
 100                 "type": "calico-ipam"
 101               },
 102               "policy": {
 103                 "type": "k8s"
 104               },
 105               "kubernetes": {
 106                 "kubeconfig": "/etc/kubernetes/admin.conf"
 107               }
 108             },
 109             {
 110               "type": "portmap",
 111               "snat": true,
 112               "capabilities": {"portMappings": true}
 113             }
 114           ]
 115         }
 116       ],
 117       "kubeconfig": "/etc/kubernetes/admin.conf"
 118     }
 119     #"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
 120 # 2. NPWG spec v1 Network Attachment Definition
 121 ---
 122 apiVersion: apiextensions.k8s.io/v1
 123 kind: CustomResourceDefinition
 124 metadata:
 125   name: network-attachment-definitions.k8s.cni.cncf.io
 126 spec:
 127   group: k8s.cni.cncf.io
 128   scope: Namespaced
 129   names:
 130     plural: network-attachment-definitions
 131     singular: network-attachment-definition
 132     kind: NetworkAttachmentDefinition
 133     shortNames:
 134     - net-attach-def
 135   versions:
 136     - name: v1
 137       served: true
 138       storage: true
 139       schema:
 140         openAPIV3Schema:
 141           type: object
 142           properties:
 143             spec:
 144               type: object
 145               properties:
 146                 config:
 147                   type: string
 148 # 3.1 Multus Cluster Role
 149 ---
 150 kind: ClusterRole
 151 apiVersion: rbac.authorization.k8s.io/v1beta1
 152 metadata:
 153   name: multus
 154 rules:
 155   - apiGroups: ["k8s.cni.cncf.io"]
 156     resources:
 157       - '*'
 158     verbs:
 159       - '*'
 160   - apiGroups:
 161       - ""
 162     resources:
 163       - pods
 164       - pods/status
 165     verbs:
 166       - get
 167       - update
 168
 169 # 3.2 Multus Cluster Role Binding
 170 ---
 171 kind: ClusterRoleBinding
 172 apiVersion: rbac.authorization.k8s.io/v1beta1
 173 metadata:
 174   name: multus
 175 roleRef:
 176   apiGroup: rbac.authorization.k8s.io
 177   kind: ClusterRole
 178   name: multus
 179 subjects:
 180 - kind: ServiceAccount
 181   name: multus
 182   namespace: kube-system
 183
 184 # 4.1 SR-IOV Device Plugin ServiceAccount
 185 ---
 186 apiVersion: v1
 187 kind: ServiceAccount
 188 metadata:
 189   name: sriov-device-plugin
 190   namespace: kube-system
 191
 192 # 4.2 Multus ServiceAccount
 193 ---
 194 apiVersion: v1
 195 kind: ServiceAccount
 196 metadata:
 197   name: multus
 198   namespace: kube-system
 199
 200 # 5.1 SR-IOV Device Plugin DaemonSet
 201 ---
 202 apiVersion: apps/v1
 203 kind: DaemonSet
 204 metadata:
 205   name: kube-sriov-device-plugin-amd64
 206   namespace: kube-system
 207   labels:
 208     tier: node
 209     app: sriovdp
 210 spec:
 211   selector:
 212     matchLabels:
 213       name: sriov-device-plugin
 214   template:
 215     metadata:
 216       labels:
 217         name: sriov-device-plugin
 218         tier: node
 219         app: sriovdp
 220     spec:
 221       hostNetwork: true
 222       hostPID: true
 223       nodeSelector:
 224         beta.kubernetes.io/arch: amd64
 225       tolerations:
 226               #- key: node-role.kubernetes.io/master
 227               #        operator: Exists
 228               #        effect: NoSchedule
 229       - operator: Exists
 230         effect: NoSchedule
 231       serviceAccountName: sriov-device-plugin
 232       containers:
 233       - name: kube-sriovdp
 234         #image: nfvpe/sriov-device-plugin
 235         image: iecedge/sriov-device-plugin-amd64
 236         imagePullPolicy: IfNotPresent
 237         args:
 238         - --log-dir=sriovdp
 239         - --log-level=10
 240         - --resource-prefix=arm.com
 241         securityContext:
 242           privileged: true
 243         volumeMounts:
 244         - name: devicesock
 245           mountPath: /var/lib/kubelet/
 246           readOnly: false
 247         - name: log
 248           mountPath: /var/log
 249         - name: config-volume
 250           mountPath: /etc/pcidp
 251       volumes:
 252         - name: devicesock
 253           hostPath:
 254             path: /var/lib/kubelet/
 255         - name: log
 256           hostPath:
 257             path: /var/log
 258         - name: config-volume
 259           configMap:
 260             name: sriovdp-config
 261             items:
 262             - key: config.json
 263               path: config.json
 264 ---
 265 apiVersion: apps/v1
 266 kind: DaemonSet
 267 metadata:
 268   name: kube-sriov-device-plugin-arm64
 269   namespace: kube-system
 270   labels:
 271     tier: node
 272     app: sriovdp
 273 spec:
 274   selector:
 275     matchLabels:
 276       name: sriov-device-plugin
 277   template:
 278     metadata:
 279       labels:
 280         name: sriov-device-plugin
 281         tier: node
 282         app: sriovdp
 283     spec:
 284       hostNetwork: true
 285       hostPID: true
 286       nodeSelector:
 287         beta.kubernetes.io/arch: arm64
 288       tolerations:
 289               #- key: node-role.kubernetes.io/master
 290               #        operator: Exists
 291               #        effect: NoSchedule
 292       - operator: Exists
 293         effect: NoSchedule
 294       serviceAccountName: sriov-device-plugin
 295       containers:
 296       - name: kube-sriovdp
 297         #image: nfvpe/sriov-device-plugin
 298         image: iecedge/sriov-device-plugin-arm64
 299         imagePullPolicy: IfNotPresent
 300         #imagePullPolicy: Never
 301         args:
 302         - --log-dir=sriovdp
 303         - --log-level=10
 304         - --resource-prefix=arm.com
 305         securityContext:
 306           privileged: true
 307         volumeMounts:
 308         - name: devicesock
 309           mountPath: /var/lib/kubelet/
 310           readOnly: false
 311         - name: log
 312           mountPath: /var/log
 313         - name: config-volume
 314           mountPath: /etc/pcidp
 315       volumes:
 316         - name: devicesock
 317           hostPath:
 318             path: /var/lib/kubelet/
 319         - name: log
 320           hostPath:
 321             path: /var/log
 322         - name: config-volume
 323           configMap:
 324             name: sriovdp-config
 325             items:
 326             - key: config.json
 327               path: config.json
 328
 329 # 5.2 SR-IOV CNI DaemonSet
 330 ---
 331 apiVersion: apps/v1
 332 kind: DaemonSet
 333 metadata:
 334   name: kube-sriov-cni-ds-amd64
 335   namespace: kube-system
 336   labels:
 337     tier: node
 338     app: sriov-cni
 339 spec:
 340   selector:
 341     matchLabels:
 342       name: sriov-cni
 343   template:
 344     metadata:
 345       labels:
 346         name: sriov-cni
 347         tier: node
 348         app: sriov-cni
 349     spec:
 350       hostNetwork: true
 351       nodeSelector:
 352         beta.kubernetes.io/arch: amd64
 353       tolerations:
 354       - key: node-role.kubernetes.io/master
 355         operator: Exists
 356         effect: NoSchedule
 357       containers:
 358       - name: kube-sriov-cni
 359         #image: nfvpe/sriov-cni:latest
 360         image: iecedge/sriov-cni-amd64:latest
 361         imagePullPolicy: IfNotPresent
 362         securityContext:
 363           privileged: true
 364         resources:
 365           requests:
 366             cpu: "100m"
 367             memory: "50Mi"
 368           limits:
 369             cpu: "100m"
 370             memory: "50Mi"
 371         volumeMounts:
 372         - name: cnibin
 373           mountPath: /host/opt/cni/bin
 374       volumes:
 375         - name: cnibin
 376           hostPath:
 377             path: /opt/cni/bin
 378 ---
 379 apiVersion: apps/v1
 380 kind: DaemonSet
 381 metadata:
 382   name: kube-sriov-cni-ds-arm64
 383   namespace: kube-system
 384   labels:
 385     tier: node
 386     app: sriov-cni
 387 spec:
 388   selector:
 389     matchLabels:
 390       name: sriov-cni
 391   template:
 392     metadata:
 393       labels:
 394         name: sriov-cni
 395         tier: node
 396         app: sriov-cni
 397     spec:
 398       hostNetwork: true
 399       nodeSelector:
 400         beta.kubernetes.io/arch: arm64
 401       tolerations:
 402               #- key: node-role.kubernetes.io/master
 403               #        operator: Exists
 404               #        effect: NoSchedule
 405       - operator: Exists
 406         effect: NoSchedule
 407       containers:
 408       - name: kube-sriov-cni
 409         #image: nfvpe/sriov-cni-arm64:latest
 410         image: iecedge/sriov-cni-arm64:latest
 411         imagePullPolicy: IfNotPresent
 412         securityContext:
 413           privileged: true
 414         resources:
 415           requests:
 416             cpu: "100m"
 417             memory: "50Mi"
 418           limits:
 419             cpu: "100m"
 420             memory: "50Mi"
 421         volumeMounts:
 422         - name: cnibin
 423           mountPath: /host/opt/cni/bin
 424       volumes:
 425         - name: cnibin
 426           hostPath:
 427             path: /opt/cni/bin
 428
 429 # 5.3 Multus DaemonSet
 430 ---
 431 apiVersion: apps/v1
 432 kind: DaemonSet
 433 metadata:
 434   name: kube-multus-ds-amd64
 435   namespace: kube-system
 436   labels:
 437     tier: node
 438     app: multus
 439     name: multus
 440 spec:
 441   selector:
 442     matchLabels:
 443       name: multus
 444   updateStrategy:
 445     type: RollingUpdate
 446   template:
 447     metadata:
 448       labels:
 449         tier: node
 450         app: multus
 451         name: multus
 452     spec:
 453       hostNetwork: true
 454       nodeSelector:
 455         beta.kubernetes.io/arch: amd64
 456       tolerations:
 457       - operator: Exists
 458         effect: NoSchedule
 459       serviceAccountName: multus
 460       containers:
 461       - name: kube-multus
 462         #image: nfvpe/multus:v3.3
 463         #- "--multus-conf-file=auto"
 464         #- "--cni-version=0.3.1"
 465         #image: nfvpe/multus:v3.4
 466         image: iecedge/multus-amd64:v3.4
 467         imagePullPolicy: IfNotPresent
 468         env:
 469         - name: KUBERNETES_NODE_NAME
 470           valueFrom:
 471             fieldRef:
 472               fieldPath: spec.nodeName
 473         command:
 474         - /bin/bash
 475         - -cex
 476         - |
 477           #!/bin/bash
 478           sed "s|__KUBERNETES_NODE_NAME__|${KUBERNETES_NODE_NAME}|g" /tmp/multus-conf/70-multus.conf.template > /tmp/multus-conf/70-multus.conf
 479           /entrypoint.sh \
 480             --multus-conf-file=/tmp/multus-conf/70-multus.conf
 481         resources:
 482           requests:
 483             cpu: "100m"
 484             memory: "50Mi"
 485           limits:
 486             cpu: "100m"
 487             memory: "50Mi"
 488         securityContext:
 489           privileged: true
 490         volumeMounts:
 491         - name: cni
 492           mountPath: /host/etc/cni/net.d
 493         - name: cnibin
 494           mountPath: /host/opt/cni/bin
 495           #- name: multus-cfg
 496           #mountPath: /tmp/multus-conf
 497           #readOnly: false
 498         - name: multus-cfg
 499           mountPath: /tmp/multus-conf/70-multus.conf.template
 500           subPath: "cni-conf.json"
 501         - name: kubernetes-cfg-dir
 502           mountPath: /etc/kubernetes
 503       volumes:
 504         - name: cni
 505           hostPath:
 506             path: /etc/cni/net.d
 507         - name: cnibin
 508           hostPath:
 509             path: /opt/cni/bin
 510             #- name: multus-cfg
 511             #configMap:
 512             #name: multus-cni-config
 513             #items:
 514             #- key: cni-conf.json
 515             #  path: 70-multus.conf.template
 516         - name: multus-cfg
 517           configMap:
 518             name: multus-cni-config
 519         - name: kubernetes-cfg-dir
 520           hostPath:
 521             path: /etc/kubernetes
 522 ---
 523 apiVersion: apps/v1
 524 kind: DaemonSet
 525 metadata:
 526   name: kube-multus-ds-arm64
 527   namespace: kube-system
 528   labels:
 529     tier: node
 530     app: multus
 531     name: multus
 532 spec:
 533   selector:
 534     matchLabels:
 535       name: multus
 536   updateStrategy:
 537     type: RollingUpdate
 538   template:
 539     metadata:
 540       labels:
 541         tier: node
 542         app: multus
 543         name: multus
 544     spec:
 545       hostNetwork: true
 546       nodeSelector:
 547         beta.kubernetes.io/arch: arm64
 548       tolerations:
 549       - operator: Exists
 550         effect: NoSchedule
 551       serviceAccountName: multus
 552       containers:
 553       - name: kube-multus
 554         #image: nfvpe/multus:v3.3
 555         #image: iecedge/multus-arm64:latest
 556         #- "--multus-conf-file=auto"
 557         #- "--cni-version=0.3.1"
 558         image: iecedge/multus-arm64:v3.4
 559         imagePullPolicy: IfNotPresent
 560         env:
 561         - name: KUBERNETES_NODE_NAME
 562           valueFrom:
 563             fieldRef:
 564               fieldPath: spec.nodeName
 565         command:
 566         - /bin/bash
 567         - -cex
 568         - |
 569           #!/bin/bash
 570           sed "s|__KUBERNETES_NODE_NAME__|${KUBERNETES_NODE_NAME}|g" /tmp/multus-conf/70-multus.conf.template > /tmp/multus-conf/70-multus.conf
 571           /entrypoint.sh \
 572             --multus-conf-file=/tmp/multus-conf/70-multus.conf
 573         resources:
 574           requests:
 575             cpu: "100m"
 576             memory: "50Mi"
 577           limits:
 578             cpu: "100m"
 579             memory: "50Mi"
 580         securityContext:
 581           privileged: true
 582         volumeMounts:
 583         - name: cni
 584           mountPath: /host/etc/cni/net.d
 585         - name: cnibin
 586           mountPath: /host/opt/cni/bin
 587           #- name: multus-cfg
 588           #mountPath: /tmp/multus-conf
 589           #readOnly: false
 590         - name: multus-cfg
 591           mountPath: /tmp/multus-conf/70-multus.conf.template
 592           subPath: "cni-conf.json"
 593         - name: kubernetes-cfg-dir
 594           mountPath: /etc/kubernetes
 595       volumes:
 596         - name: cni
 597           hostPath:
 598             path: /etc/cni/net.d
 599         - name: cnibin
 600           hostPath:
 601             path: /opt/cni/bin
 602             #- name: multus-cfg
 603             #configMap:
 604             #name: multus-cni-config
 605             #items:
 606             #- key: cni-conf.json
 607             #  path: 70-multus.conf.template
 608         - name: multus-cfg
 609           configMap:
 610             name: multus-cni-config
 611         - name: kubernetes-cfg-dir
 612           hostPath:
 613             path: /etc/kubernetes
 614