--- /dev/null
+# yamllint disable
+# This yaml file contains necessary configuration to setup
+# a demo environment for Multus + SR-IOV, the config includes
+# the following pieces:
+# 1. Multus ConfigMap
+# 2. Network Plumbing Working Group Spec Version 1 CustomerResourceDefinition
+# 3. Multus ClusterRole & ClusterRoleBinding
+# 4. Multus & SR-IOV Device Plugin ServiceAccounts
+# 5. Multus & SR-IOV Device Plugin & SR-IOV CNI DaemonSets
+
+# Note: This yaml file will not create customer SR-IOV CRD
+# which will be specified in Pod spec annotation. Below is
+# an example of SR-IOV CRD:
+#
+# apiVersion: "k8s.cni.cncf.io/v1"
+# kind: NetworkAttachmentDefinition
+# metadata:
+# name: sriov-net1
+# annotations:
+# k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+# spec:
+# config: '{
+# "type": "sriov",
+# "name": "sriov-network",
+# "ipam": {
+# "type": "host-local",
+# "subnet": "10.56.217.0/24",
+# "routes": [{
+# "dst": "0.0.0.0/0"
+# }],
+# "gateway": "10.56.217.1"
+# }
+# }'
+
+# An example of Pod spec using above SR-IOV CRD:
+#
+# apiVersion: v1
+# kind: Pod
+# metadata:
+# name: testpod1
+# labels:
+# env: test
+# annotations:
+# k8s.v1.cni.cncf.io/networks: sriov-net1
+# spec:
+# containers:
+# - name: appcntr1
+# image: centos/tools
+# imagePullPolicy: IfNotPresent
+# command: [ "/bin/bash", "-c", "--" ]
+# args: [ "while true; do sleep 300000; done;" ]
+# resources:
+# requests:
+# intel.com/sriov: '1'
+# limits:
+# intel.com/sriov: '1'
+
+
+# --------------------------------------------------------------------
+
+# 1. Multus ConfigMap
+#
+# This configMap assumes that:
+# - Kubeconfig file is located at "/etc/kubernetes/admin.conf" on host
+# - Default master plugin for Multus is set to flannel
+#
+# Note: If either of above is not True in your environment
+# make sure they are properly set to the corrent values.
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: multus-cni-config
+ namespace: kube-system
+ labels:
+ tier: node
+ app: multus
+data:
+ cni-conf.json: |
+ {
+ "name": "multus-cni-network",
+ "type": "multus",
+ "capabilities": {
+ "portMappings": true
+ },
+ "delegates": [
+ {
+ "cniVersion": "0.3.1",
+ "name": "default-cni-network",
+ "plugins": [
+ {
+ "name": "k8s-pod-network",
+ "cniVersion": "0.3.0",
+ "type": "calico",
+ "log_level": "info",
+ "datastore_type": "kubernetes",
+ "nodename": "__KUBERNETES_NODE_NAME__",
+ "mtu": 1440,
+ "ipam": {
+ "type": "calico-ipam"
+ },
+ "policy": {
+ "type": "k8s"
+ },
+ "kubernetes": {
+ "kubeconfig": "/etc/kubernetes/admin.conf"
+ }
+ },
+ {
+ "type": "portmap",
+ "snat": true,
+ "capabilities": {"portMappings": true}
+ }
+ ]
+ }
+ ],
+ "kubeconfig": "/etc/kubernetes/admin.conf"
+ }
+ #"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+# 2. NPWG spec v1 Network Attachment Definition
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+ group: k8s.cni.cncf.io
+ version: v1
+ scope: Namespaced
+ names:
+ plural: network-attachment-definitions
+ singular: network-attachment-definition
+ kind: NetworkAttachmentDefinition
+ shortNames:
+ - net-attach-def
+ validation:
+ openAPIV3Schema:
+ properties:
+ spec:
+ properties:
+ config:
+ type: string
+
+
+# 3.1 Multus Cluster Role
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: multus
+rules:
+ - apiGroups: ["k8s.cni.cncf.io"]
+ resources:
+ - '*'
+ verbs:
+ - '*'
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ - pods/status
+ verbs:
+ - get
+ - update
+
+# 3.2 Multus Cluster Role Binding
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: multus
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: multus
+subjects:
+- kind: ServiceAccount
+ name: multus
+ namespace: kube-system
+
+# 4.1 SR-IOV Device Plugin ServiceAccount
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: sriov-device-plugin
+ namespace: kube-system
+
+# 4.2 Multus ServiceAccount
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: multus
+ namespace: kube-system
+
+# 5.1 SR-IOV Device Plugin DaemonSet
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-sriov-device-plugin-amd64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: sriovdp
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: sriovdp
+ spec:
+ hostNetwork: true
+ hostPID: true
+ nodeSelector:
+ beta.kubernetes.io/arch: amd64
+ tolerations:
+ #- key: node-role.kubernetes.io/master
+ # operator: Exists
+ # effect: NoSchedule
+ - operator: Exists
+ effect: NoSchedule
+ serviceAccountName: sriov-device-plugin
+ containers:
+ - name: kube-sriovdp
+ image: nfvpe/sriov-device-plugin
+ imagePullPolicy: IfNotPresent
+ args:
+ - --log-dir=sriovdp
+ - --log-level=10
+ - --resource-prefix=arm.com
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: devicesock
+ mountPath: /var/lib/kubelet/
+ readOnly: false
+ - name: log
+ mountPath: /var/log
+ - name: config-volume
+ mountPath: /etc/pcidp
+ volumes:
+ - name: devicesock
+ hostPath:
+ path: /var/lib/kubelet/
+ - name: log
+ hostPath:
+ path: /var/log
+ - name: config-volume
+ configMap:
+ name: sriovdp-config
+ items:
+ - key: config.json
+ path: config.json
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-sriov-device-plugin-arm64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: sriovdp
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: sriovdp
+ spec:
+ hostNetwork: true
+ hostPID: true
+ nodeSelector:
+ beta.kubernetes.io/arch: arm64
+ tolerations:
+ #- key: node-role.kubernetes.io/master
+ # operator: Exists
+ # effect: NoSchedule
+ - operator: Exists
+ effect: NoSchedule
+ serviceAccountName: sriov-device-plugin
+ containers:
+ - name: kube-sriovdp
+ #image: nfvpe/sriov-device-plugin
+ image: iecedge/sriov-device-plugin-arm64
+ imagePullPolicy: IfNotPresent
+ #imagePullPolicy: Never
+ args:
+ - --log-dir=sriovdp
+ - --log-level=10
+ - --resource-prefix=arm.com
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: devicesock
+ mountPath: /var/lib/kubelet/
+ readOnly: false
+ - name: log
+ mountPath: /var/log
+ - name: config-volume
+ mountPath: /etc/pcidp
+ volumes:
+ - name: devicesock
+ hostPath:
+ path: /var/lib/kubelet/
+ - name: log
+ hostPath:
+ path: /var/log
+ - name: config-volume
+ configMap:
+ name: sriovdp-config
+ items:
+ - key: config.json
+ path: config.json
+
+# 5.2 SR-IOV CNI DaemonSet
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-sriov-cni-ds-amd64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: sriov-cni
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: sriov-cni
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ beta.kubernetes.io/arch: amd64
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ containers:
+ - name: kube-sriov-cni
+ image: nfvpe/sriov-cni:latest
+ imagePullPolicy: IfNotPresent
+ securityContext:
+ privileged: true
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50Mi"
+ limits:
+ cpu: "100m"
+ memory: "50Mi"
+ volumeMounts:
+ - name: cnibin
+ mountPath: /host/opt/cni/bin
+ volumes:
+ - name: cnibin
+ hostPath:
+ path: /opt/cni/bin
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-sriov-cni-ds-arm64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: sriov-cni
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: sriov-cni
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ beta.kubernetes.io/arch: arm64
+ tolerations:
+ #- key: node-role.kubernetes.io/master
+ # operator: Exists
+ # effect: NoSchedule
+ - operator: Exists
+ effect: NoSchedule
+ containers:
+ - name: kube-sriov-cni
+ #image: nfvpe/sriov-cni-arm64:latest
+ image: iecedge/sriov-cni-arm64:latest
+ imagePullPolicy: IfNotPresent
+ securityContext:
+ privileged: true
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50Mi"
+ limits:
+ cpu: "100m"
+ memory: "50Mi"
+ volumeMounts:
+ - name: cnibin
+ mountPath: /host/opt/cni/bin
+ volumes:
+ - name: cnibin
+ hostPath:
+ path: /opt/cni/bin
+
+# 5.3 Multus DaemonSet
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: kube-multus-ds-amd64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: multus
+ name: multus
+spec:
+ selector:
+ matchLabels:
+ name: multus
+ updateStrategy:
+ type: RollingUpdate
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: multus
+ name: multus
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ beta.kubernetes.io/arch: amd64
+ tolerations:
+ - operator: Exists
+ effect: NoSchedule
+ serviceAccountName: multus
+ containers:
+ - name: kube-multus
+ #image: nfvpe/multus:v3.3
+ #- "--multus-conf-file=auto"
+ #- "--cni-version=0.3.1"
+ image: nfvpe/multus:v3.4
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: KUBERNETES_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ command:
+ - /bin/bash
+ - -cex
+ - |
+ #!/bin/bash
+ sed "s|__KUBERNETES_NODE_NAME__|${KUBERNETES_NODE_NAME}|g" /tmp/multus-conf/70-multus.conf.template > /tmp/multus-conf/70-multus.conf
+ /entrypoint.sh \
+ --multus-conf-file=/tmp/multus-conf/70-multus.conf
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50Mi"
+ limits:
+ cpu: "100m"
+ memory: "50Mi"
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: cni
+ mountPath: /host/etc/cni/net.d
+ - name: cnibin
+ mountPath: /host/opt/cni/bin
+ #- name: multus-cfg
+ #mountPath: /tmp/multus-conf
+ #readOnly: false
+ - name: multus-cfg
+ mountPath: /tmp/multus-conf/70-multus.conf.template
+ subPath: "cni-conf.json"
+ - name: kubernetes-cfg-dir
+ mountPath: /etc/kubernetes
+ volumes:
+ - name: cni
+ hostPath:
+ path: /etc/cni/net.d
+ - name: cnibin
+ hostPath:
+ path: /opt/cni/bin
+ #- name: multus-cfg
+ #configMap:
+ #name: multus-cni-config
+ #items:
+ #- key: cni-conf.json
+ # path: 70-multus.conf.template
+ - name: multus-cfg
+ configMap:
+ name: multus-cni-config
+ - name: kubernetes-cfg-dir
+ hostPath:
+ path: /etc/kubernetes
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: kube-multus-ds-arm64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: multus
+ name: multus
+spec:
+ selector:
+ matchLabels:
+ name: multus
+ updateStrategy:
+ type: RollingUpdate
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: multus
+ name: multus
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ beta.kubernetes.io/arch: arm64
+ tolerations:
+ - operator: Exists
+ effect: NoSchedule
+ serviceAccountName: multus
+ containers:
+ - name: kube-multus
+ #image: nfvpe/multus:v3.3
+ #image: iecedge/multus-arm64:latest
+ #- "--multus-conf-file=auto"
+ #- "--cni-version=0.3.1"
+ image: iecedge/multus-arm64:v3.4
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: KUBERNETES_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ command:
+ - /bin/bash
+ - -cex
+ - |
+ #!/bin/bash
+ sed "s|__KUBERNETES_NODE_NAME__|${KUBERNETES_NODE_NAME}|g" /tmp/multus-conf/70-multus.conf.template > /tmp/multus-conf/70-multus.conf
+ /entrypoint.sh \
+ --multus-conf-file=/tmp/multus-conf/70-multus.conf
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50Mi"
+ limits:
+ cpu: "100m"
+ memory: "50Mi"
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: cni
+ mountPath: /host/etc/cni/net.d
+ - name: cnibin
+ mountPath: /host/opt/cni/bin
+ #- name: multus-cfg
+ #mountPath: /tmp/multus-conf
+ #readOnly: false
+ - name: multus-cfg
+ mountPath: /tmp/multus-conf/70-multus.conf.template
+ subPath: "cni-conf.json"
+ - name: kubernetes-cfg-dir
+ mountPath: /etc/kubernetes
+ volumes:
+ - name: cni
+ hostPath:
+ path: /etc/cni/net.d
+ - name: cnibin
+ hostPath:
+ path: /opt/cni/bin
+ #- name: multus-cfg
+ #configMap:
+ #name: multus-cni-config
+ #items:
+ #- key: cni-conf.json
+ # path: 70-multus.conf.template
+ - name: multus-cfg
+ configMap:
+ name: multus-cni-config
+ - name: kubernetes-cfg-dir
+ hostPath:
+ path: /etc/kubernetes
+
Code Review / iec.git / blobdiff