src/foundation/scripts/cni/multus/multus-sriov-calico/multus-sriov-calico-daemonsets.yaml

   1 # yamllint disable
   2 # This yaml file contains necessary configuration to setup
   3 # a demo environment for Multus + SR-IOV, the config includes
   4 # the following pieces:
   5 # 1. Multus ConfigMap
   6 # 2. Network Plumbing Working Group Spec Version 1 CustomerResourceDefinition
   7 # 3. Multus ClusterRole & ClusterRoleBinding
   8 # 4. Multus & SR-IOV Device Plugin ServiceAccounts
   9 # 5. Multus & SR-IOV Device Plugin & SR-IOV CNI DaemonSets
  10
  11 # Note: This yaml file will not create customer SR-IOV CRD
  12 # which will be specified in Pod spec annotation. Below is
  13 # an example of SR-IOV CRD:
  14 #
  15 # apiVersion: "k8s.cni.cncf.io/v1"
  16 # kind: NetworkAttachmentDefinition
  17 # metadata:
  18 #   name: sriov-net1
  19 #   annotations:
  20 #     k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
  21 # spec:
  22 #   config: '{
  23 #        "type": "sriov",
  24 #        "name": "sriov-network",
  25 #        "ipam": {
  26 #                "type": "host-local",
  27 #                "subnet": "10.56.217.0/24",
  28 #                "routes": [{
  29 #                        "dst": "0.0.0.0/0"
  30 #                }],
  31 #                "gateway": "10.56.217.1"
  32 #        }
  33 #   }'
  34
  35 # An example of Pod spec using above SR-IOV CRD:
  36 #
  37 # apiVersion: v1
  38 # kind: Pod
  39 # metadata:
  40 #   name: testpod1
  41 #   labels:
  42 #     env: test
  43 #   annotations:
  44 #     k8s.v1.cni.cncf.io/networks: sriov-net1
  45 # spec:
  46 #   containers:
  47 #   - name: appcntr1
  48 #     image: centos/tools
  49 #     imagePullPolicy: IfNotPresent
  50 #     command: [ "/bin/bash", "-c", "--" ]
  51 #     args: [ "while true; do sleep 300000; done;" ]
  52 #     resources:
  53 #       requests:
  54 #         intel.com/sriov: '1'
  55 #       limits:
  56 #        intel.com/sriov: '1'
  57
  58
  59 # --------------------------------------------------------------------
  60
  61 # 1. Multus ConfigMap
  62 #
  63 # This configMap assumes that:
  64 # - Kubeconfig file is located at "/etc/kubernetes/admin.conf" on host
  65 # - Default master plugin for Multus is set to flannel
  66 #
  67 # Note: If either of above is not True in your environment
  68 # make sure they are properly set to the corrent values.
  69 ---
  70 kind: ConfigMap
  71 apiVersion: v1
  72 metadata:
  73   name: multus-cni-config
  74   namespace: kube-system
  75   labels:
  76     tier: node
  77     app: multus
  78 data:
  79   cni-conf.json: |
  80     {
  81       "name": "multus-cni-network",
  82       "type": "multus",
  83       "capabilities": {
  84         "portMappings": true
  85       },
  86       "delegates": [
  87         {
  88           "cniVersion": "0.3.1",
  89           "name": "default-cni-network",
  90           "plugins": [
  91             {
  92               "name": "k8s-pod-network",
  93               "cniVersion": "0.3.0",
  94               "type": "calico",
  95               "log_level": "info",
  96               "datastore_type": "kubernetes",
  97               "nodename": "__KUBERNETES_NODE_NAME__",
  98               "mtu": 1440,
  99               "ipam": {
 100                 "type": "calico-ipam"
 101               },
 102               "policy": {
 103                 "type": "k8s"
 104               },
 105               "kubernetes": {
 106                 "kubeconfig": "/etc/kubernetes/admin.conf"
 107               }
 108             },
 109             {
 110               "type": "portmap",
 111               "snat": true,
 112               "capabilities": {"portMappings": true}
 113             }
 114           ]
 115         }
 116       ],
 117       "kubeconfig": "/etc/kubernetes/admin.conf"
 118     }
 119     #"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
 120 # 2. NPWG spec v1 Network Attachment Definition
 121 ---
 122 apiVersion: apiextensions.k8s.io/v1beta1
 123 kind: CustomResourceDefinition
 124 metadata:
 125   name: network-attachment-definitions.k8s.cni.cncf.io
 126 spec:
 127   group: k8s.cni.cncf.io
 128   version: v1
 129   scope: Namespaced
 130   names:
 131     plural: network-attachment-definitions
 132     singular: network-attachment-definition
 133     kind: NetworkAttachmentDefinition
 134     shortNames:
 135     - net-attach-def
 136   validation:
 137     openAPIV3Schema:
 138       properties:
 139         spec:
 140           properties:
 141             config:
 142                  type: string
 143
 144
 145 # 3.1 Multus Cluster Role
 146 ---
 147 kind: ClusterRole
 148 apiVersion: rbac.authorization.k8s.io/v1beta1
 149 metadata:
 150   name: multus
 151 rules:
 152   - apiGroups: ["k8s.cni.cncf.io"]
 153     resources:
 154       - '*'
 155     verbs:
 156       - '*'
 157   - apiGroups:
 158       - ""
 159     resources:
 160       - pods
 161       - pods/status
 162     verbs:
 163       - get
 164       - update
 165
 166 # 3.2 Multus Cluster Role Binding
 167 ---
 168 kind: ClusterRoleBinding
 169 apiVersion: rbac.authorization.k8s.io/v1beta1
 170 metadata:
 171   name: multus
 172 roleRef:
 173   apiGroup: rbac.authorization.k8s.io
 174   kind: ClusterRole
 175   name: multus
 176 subjects:
 177 - kind: ServiceAccount
 178   name: multus
 179   namespace: kube-system
 180
 181 # 4.1 SR-IOV Device Plugin ServiceAccount
 182 ---
 183 apiVersion: v1
 184 kind: ServiceAccount
 185 metadata:
 186   name: sriov-device-plugin
 187   namespace: kube-system
 188
 189 # 4.2 Multus ServiceAccount
 190 ---
 191 apiVersion: v1
 192 kind: ServiceAccount
 193 metadata:
 194   name: multus
 195   namespace: kube-system
 196
 197 # 5.1 SR-IOV Device Plugin DaemonSet
 198 ---
 199 apiVersion: extensions/v1beta1
 200 kind: DaemonSet
 201 metadata:
 202   name: kube-sriov-device-plugin-amd64
 203   namespace: kube-system
 204   labels:
 205     tier: node
 206     app: sriovdp
 207 spec:
 208   template:
 209     metadata:
 210       labels:
 211         tier: node
 212         app: sriovdp
 213     spec:
 214       hostNetwork: true
 215       hostPID: true
 216       nodeSelector:
 217         beta.kubernetes.io/arch: amd64
 218       tolerations:
 219               #- key: node-role.kubernetes.io/master
 220               #        operator: Exists
 221               #        effect: NoSchedule
 222       - operator: Exists
 223         effect: NoSchedule
 224       serviceAccountName: sriov-device-plugin
 225       containers:
 226       - name: kube-sriovdp
 227         image: nfvpe/sriov-device-plugin
 228         imagePullPolicy: IfNotPresent
 229         args:
 230         - --log-dir=sriovdp
 231         - --log-level=10
 232         - --resource-prefix=arm.com
 233         securityContext:
 234           privileged: true
 235         volumeMounts:
 236         - name: devicesock
 237           mountPath: /var/lib/kubelet/
 238           readOnly: false
 239         - name: log
 240           mountPath: /var/log
 241         - name: config-volume
 242           mountPath: /etc/pcidp
 243       volumes:
 244         - name: devicesock
 245           hostPath:
 246             path: /var/lib/kubelet/
 247         - name: log
 248           hostPath:
 249             path: /var/log
 250         - name: config-volume
 251           configMap:
 252             name: sriovdp-config
 253             items:
 254             - key: config.json
 255               path: config.json
 256 ---
 257 apiVersion: extensions/v1beta1
 258 kind: DaemonSet
 259 metadata:
 260   name: kube-sriov-device-plugin-arm64
 261   namespace: kube-system
 262   labels:
 263     tier: node
 264     app: sriovdp
 265 spec:
 266   template:
 267     metadata:
 268       labels:
 269         tier: node
 270         app: sriovdp
 271     spec:
 272       hostNetwork: true
 273       hostPID: true
 274       nodeSelector:
 275         beta.kubernetes.io/arch: arm64
 276       tolerations:
 277               #- key: node-role.kubernetes.io/master
 278               #        operator: Exists
 279               #        effect: NoSchedule
 280       - operator: Exists
 281         effect: NoSchedule
 282       serviceAccountName: sriov-device-plugin
 283       containers:
 284       - name: kube-sriovdp
 285         #image: nfvpe/sriov-device-plugin
 286         image: iecedge/sriov-device-plugin-arm64
 287         imagePullPolicy: IfNotPresent
 288         #imagePullPolicy: Never
 289         args:
 290         - --log-dir=sriovdp
 291         - --log-level=10
 292         - --resource-prefix=arm.com
 293         securityContext:
 294           privileged: true
 295         volumeMounts:
 296         - name: devicesock
 297           mountPath: /var/lib/kubelet/
 298           readOnly: false
 299         - name: log
 300           mountPath: /var/log
 301         - name: config-volume
 302           mountPath: /etc/pcidp
 303       volumes:
 304         - name: devicesock
 305           hostPath:
 306             path: /var/lib/kubelet/
 307         - name: log
 308           hostPath:
 309             path: /var/log
 310         - name: config-volume
 311           configMap:
 312             name: sriovdp-config
 313             items:
 314             - key: config.json
 315               path: config.json
 316
 317 # 5.2 SR-IOV CNI DaemonSet
 318 ---
 319 apiVersion: extensions/v1beta1
 320 kind: DaemonSet
 321 metadata:
 322   name: kube-sriov-cni-ds-amd64
 323   namespace: kube-system
 324   labels:
 325     tier: node
 326     app: sriov-cni
 327 spec:
 328   template:
 329     metadata:
 330       labels:
 331         tier: node
 332         app: sriov-cni
 333     spec:
 334       hostNetwork: true
 335       nodeSelector:
 336         beta.kubernetes.io/arch: amd64
 337       tolerations:
 338       - key: node-role.kubernetes.io/master
 339         operator: Exists
 340         effect: NoSchedule
 341       containers:
 342       - name: kube-sriov-cni
 343         image: nfvpe/sriov-cni:latest
 344         imagePullPolicy: IfNotPresent
 345         securityContext:
 346           privileged: true
 347         resources:
 348           requests:
 349             cpu: "100m"
 350             memory: "50Mi"
 351           limits:
 352             cpu: "100m"
 353             memory: "50Mi"
 354         volumeMounts:
 355         - name: cnibin
 356           mountPath: /host/opt/cni/bin
 357       volumes:
 358         - name: cnibin
 359           hostPath:
 360             path: /opt/cni/bin
 361 ---
 362 apiVersion: extensions/v1beta1
 363 kind: DaemonSet
 364 metadata:
 365   name: kube-sriov-cni-ds-arm64
 366   namespace: kube-system
 367   labels:
 368     tier: node
 369     app: sriov-cni
 370 spec:
 371   template:
 372     metadata:
 373       labels:
 374         tier: node
 375         app: sriov-cni
 376     spec:
 377       hostNetwork: true
 378       nodeSelector:
 379         beta.kubernetes.io/arch: arm64
 380       tolerations:
 381               #- key: node-role.kubernetes.io/master
 382               #        operator: Exists
 383               #        effect: NoSchedule
 384       - operator: Exists
 385         effect: NoSchedule
 386       containers:
 387       - name: kube-sriov-cni
 388         #image: nfvpe/sriov-cni-arm64:latest
 389         image: iecedge/sriov-cni-arm64:latest
 390         imagePullPolicy: IfNotPresent
 391         securityContext:
 392           privileged: true
 393         resources:
 394           requests:
 395             cpu: "100m"
 396             memory: "50Mi"
 397           limits:
 398             cpu: "100m"
 399             memory: "50Mi"
 400         volumeMounts:
 401         - name: cnibin
 402           mountPath: /host/opt/cni/bin
 403       volumes:
 404         - name: cnibin
 405           hostPath:
 406             path: /opt/cni/bin
 407
 408 # 5.3 Multus DaemonSet
 409 ---
 410 apiVersion: apps/v1
 411 kind: DaemonSet
 412 metadata:
 413   name: kube-multus-ds-amd64
 414   namespace: kube-system
 415   labels:
 416     tier: node
 417     app: multus
 418     name: multus
 419 spec:
 420   selector:
 421     matchLabels:
 422       name: multus
 423   updateStrategy:
 424     type: RollingUpdate
 425   template:
 426     metadata:
 427       labels:
 428         tier: node
 429         app: multus
 430         name: multus
 431     spec:
 432       hostNetwork: true
 433       nodeSelector:
 434         beta.kubernetes.io/arch: amd64
 435       tolerations:
 436       - operator: Exists
 437         effect: NoSchedule
 438       serviceAccountName: multus
 439       containers:
 440       - name: kube-multus
 441         #image: nfvpe/multus:v3.3
 442         #- "--multus-conf-file=auto"
 443         #- "--cni-version=0.3.1"
 444         image: nfvpe/multus:v3.4
 445         imagePullPolicy: IfNotPresent
 446         env:
 447         - name: KUBERNETES_NODE_NAME
 448           valueFrom:
 449             fieldRef:
 450               fieldPath: spec.nodeName
 451         command:
 452         - /bin/bash
 453         - -cex
 454         - |
 455           #!/bin/bash
 456           sed "s|__KUBERNETES_NODE_NAME__|${KUBERNETES_NODE_NAME}|g" /tmp/multus-conf/70-multus.conf.template > /tmp/multus-conf/70-multus.conf
 457           /entrypoint.sh \
 458             --multus-conf-file=/tmp/multus-conf/70-multus.conf
 459         resources:
 460           requests:
 461             cpu: "100m"
 462             memory: "50Mi"
 463           limits:
 464             cpu: "100m"
 465             memory: "50Mi"
 466         securityContext:
 467           privileged: true
 468         volumeMounts:
 469         - name: cni
 470           mountPath: /host/etc/cni/net.d
 471         - name: cnibin
 472           mountPath: /host/opt/cni/bin
 473           #- name: multus-cfg
 474           #mountPath: /tmp/multus-conf
 475           #readOnly: false
 476         - name: multus-cfg
 477           mountPath: /tmp/multus-conf/70-multus.conf.template
 478           subPath: "cni-conf.json"
 479         - name: kubernetes-cfg-dir
 480           mountPath: /etc/kubernetes
 481       volumes:
 482         - name: cni
 483           hostPath:
 484             path: /etc/cni/net.d
 485         - name: cnibin
 486           hostPath:
 487             path: /opt/cni/bin
 488             #- name: multus-cfg
 489             #configMap:
 490             #name: multus-cni-config
 491             #items:
 492             #- key: cni-conf.json
 493             #  path: 70-multus.conf.template
 494         - name: multus-cfg
 495           configMap:
 496             name: multus-cni-config
 497         - name: kubernetes-cfg-dir
 498           hostPath:
 499             path: /etc/kubernetes
 500 ---
 501 apiVersion: apps/v1
 502 kind: DaemonSet
 503 metadata:
 504   name: kube-multus-ds-arm64
 505   namespace: kube-system
 506   labels:
 507     tier: node
 508     app: multus
 509     name: multus
 510 spec:
 511   selector:
 512     matchLabels:
 513       name: multus
 514   updateStrategy:
 515     type: RollingUpdate
 516   template:
 517     metadata:
 518       labels:
 519         tier: node
 520         app: multus
 521         name: multus
 522     spec:
 523       hostNetwork: true
 524       nodeSelector:
 525         beta.kubernetes.io/arch: arm64
 526       tolerations:
 527       - operator: Exists
 528         effect: NoSchedule
 529       serviceAccountName: multus
 530       containers:
 531       - name: kube-multus
 532         #image: nfvpe/multus:v3.3
 533         #image: iecedge/multus-arm64:latest
 534         #- "--multus-conf-file=auto"
 535         #- "--cni-version=0.3.1"
 536         image: iecedge/multus-arm64:v3.4
 537         imagePullPolicy: IfNotPresent
 538         env:
 539         - name: KUBERNETES_NODE_NAME
 540           valueFrom:
 541             fieldRef:
 542               fieldPath: spec.nodeName
 543         command:
 544         - /bin/bash
 545         - -cex
 546         - |
 547           #!/bin/bash
 548           sed "s|__KUBERNETES_NODE_NAME__|${KUBERNETES_NODE_NAME}|g" /tmp/multus-conf/70-multus.conf.template > /tmp/multus-conf/70-multus.conf
 549           /entrypoint.sh \
 550             --multus-conf-file=/tmp/multus-conf/70-multus.conf
 551         resources:
 552           requests:
 553             cpu: "100m"
 554             memory: "50Mi"
 555           limits:
 556             cpu: "100m"
 557             memory: "50Mi"
 558         securityContext:
 559           privileged: true
 560         volumeMounts:
 561         - name: cni
 562           mountPath: /host/etc/cni/net.d
 563         - name: cnibin
 564           mountPath: /host/opt/cni/bin
 565           #- name: multus-cfg
 566           #mountPath: /tmp/multus-conf
 567           #readOnly: false
 568         - name: multus-cfg
 569           mountPath: /tmp/multus-conf/70-multus.conf.template
 570           subPath: "cni-conf.json"
 571         - name: kubernetes-cfg-dir
 572           mountPath: /etc/kubernetes
 573       volumes:
 574         - name: cni
 575           hostPath:
 576             path: /etc/cni/net.d
 577         - name: cnibin
 578           hostPath:
 579             path: /opt/cni/bin
 580             #- name: multus-cfg
 581             #configMap:
 582             #name: multus-cni-config
 583             #items:
 584             #- key: cni-conf.json
 585             #  path: 70-multus.conf.template
 586         - name: multus-cfg
 587           configMap:
 588             name: multus-cni-config
 589         - name: kubernetes-cfg-dir
 590           hostPath:
 591             path: /etc/kubernetes
 592