+kud-plugin-addons provides the scripts and yaml files to install kud-addons on working node which includes: device plugins, nfd, prometheus, rook etc.
+1. device plugin:
+(1) driver: include scripts to install driver on working node
+collect_XX.sh: invoked by collect.sh in build machine to generate the driver installation package (e.g. qat_driver-0.1.tar.gz etc.), the package will be installed on /opt/icn/driver in icn infra-local-controller. BPA can install this package on cluster node, then install the driver with below commands:
+$ tar xzvf qat_driver-0.1.tar.gz
+$ cd qat_driver
+$ bash install.sh
+
+(2) yaml: include the yaml file to install k8s device plugin
+collect_XX.sh: invoked by collect.sh in build machine to generate the k8s device plugin installation package (e.g. qat_yaml-0.1.tar.gz etc.), the package will be installed on /opt/icn/yaml in icn infra-local-controller. BPA can install this package on cluster node, then install the device plugin with below commands(it assumed that k8s cluster had been installed by kud):
+$ tar xzvf qat_yaml-0.1.tar.gz
+$ cd qat_yaml
+$ bash install.sh
+
+(3) test: include the test yaml file with sample workloads to use the device plugin
+
+2. nfd
+3. prometheus
+4. rook
+
+# Precondition:
+# (1) QAT device(e.g. 37c8) installed
+# (2) Enable grub with "intel_iommu=on iommu=pt"
+# (3) Driver install script is put at the same folder of QAT build target
+
+# QAT package: https://01.org/zh/intel-quick-assist-technology/downloads
+
+# install qat driver
+cd driver
+./install_qat.sh
+
+# install qat device plugin
+# pre-pull local build docker image: intel-qat-plugin:devel
+cd yaml
+cat qat_plugin_default_configmap.yaml | kubectl apply -f -
+cat qat_plugin_privileges.yaml | kubectl apply -f -
+
+# test
+# pre-pull local build docker image: crypto-perf:devel
+cd test
+cat test.yaml | kubectl apply -f -
+kubectl exec -it dpdk2 bash
+./dpdk-test-crypto-perf -l 6-7 -w $QAT1 -- --ptest throughput\
+ --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc\
+ --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000\
+ --burst-sz 32 --buffer-sz 64
--- /dev/null
+#!/bin/bash
+
+# usage: collect_qat_driver.sh [target]
+
+set -ex
+
+if [ $# -ne 1 ] ; then
+ echo "Please input the target folder!"
+ exit 0
+fi
+
+VER="0.1"
+MKDIR_P="mkdir -p"
+target=$1
+package=qat1.7.l.4.6.0-00025
+
+# download driver source package
+if [ ! -e /tmp/$package.tar.gz ]; then
+ wget -P /tmp https://01.org/sites/default/files/downloads/$package.tar.gz
+fi
+cp /tmp/$package.tar.gz .
+
+# compile
+$MKDIR_P $package
+tar xzvf $package.tar.gz -C $package
+pushd `pwd`
+cd $package
+./configure --enable-icp-sriov=host
+make
+popd
+
+# copy to target
+$MKDIR_P qat_driver
+cp -r $package/build/* qat_driver/
+cp install_qat.sh qat_driver/install.sh
+cp qat qat_driver/
+
+if [ ! -d $target/driver ]; then
+ $MKDIR_P $target/driver;
+fi;
+
+tar czvf $target/driver/qat_driver-$VER.tar.gz qat_driver/
+
+# clear
+rm -rf $package
+rm -rf qat_driver
+rm $package.tar.gz
--- /dev/null
+#!/bin/bash
+
+# Precondition:
+# QAT device installed, such as lspci | grep 37c8
+# Enable grub with "intel_iommu=on iommu=pt"
+
+ROOT=
+
+MV=mv
+RM=rm
+ECHO=echo
+SLEEP=sleep
+INSTALL="/usr/bin/install -c"
+MKDIR_P="mkdir -p"
+libdir=/usr/local/lib
+bindir=/usr/local/bin
+
+am__append_1="drivers/crypto/qat/qat_dh895xcc/qat_dh895xcc.ko\
+ drivers/crypto/qat/qat_dh895xccvf/qat_dh895xccvf.ko"
+
+am__append_2="qat_895xcc.bin qat_895xcc_mmp.bin"
+am__append_3="dh895xcc_dev0.conf dh895xcc_dev1.conf dh895xccvf_dev0.conf.vm"
+
+# Kernel modules list
+KO_MODULES_LIST="drivers/crypto/qat/qat_common/intel_qat.ko \
+ drivers/crypto/qat/qat_c62x/qat_c62x.ko \
+ drivers/crypto/qat/qat_c62xvf/qat_c62xvf.ko \
+ drivers/crypto/qat/qat_d15xx/qat_d15xx.ko \
+ drivers/crypto/qat/qat_d15xxvf/qat_d15xxvf.ko \
+ drivers/crypto/qat/qat_c3xxx/qat_c3xxx.ko \
+ drivers/crypto/qat/qat_c3xxxvf/qat_c3xxxvf.ko $am__append_1"
+
+# Firmwares list
+BIN_LIST="qat_c3xxx.bin qat_c3xxx_mmp.bin qat_c62x.bin \
+ qat_c62x_mmp.bin qat_mmp.bin qat_d15xx.bin qat_d15xx_mmp.bin \
+ $am__append_2"
+CONFIG_LIST="c3xxx_dev0.conf \
+ c3xxxvf_dev0.conf.vm \
+ c6xx_dev0.conf \
+ c6xx_dev1.conf \
+ c6xx_dev2.conf \
+ c6xxvf_dev0.conf.vm \
+ d15xx_dev0.conf \
+ d15xxpf_dev0.conf \
+ d15xxvf_dev0.conf.vm \
+ $am__append_3"
+
+QAT_DH895XCC_NUM_VFS=32
+QAT_DHC62X_NUM_VFS=16
+QAT_DHD15XX_NUM_VFS=16
+QAT_DHC3XXX_NUM_VFS=16
+
+# Device information variables
+INTEL_VENDORID="8086"
+DH895_DEVICE_NUMBER="0435"
+DH895_DEVICE_NUMBER_VM="0443"
+C62X_DEVICE_NUMBER="37c8"
+C62X_DEVICE_NUMBER_VM="37c9"
+D15XX_DEVICE_NUMBER="6f54"
+D15XX_DEVICE_NUMBER_VM="6f55"
+C3XXX_DEVICE_NUMBER="19e2"
+C3XXX_DEVICE_NUMBER_VM="19e3"
+numC62xDevice=`lspci -vnd 8086: | egrep -c "37c8|37c9"`
+numD15xxDevice=`lspci -vnd 8086: | egrep -c "6f54|6f55"`
+numDh895xDevice=`lspci -vnd 8086: | egrep -c "0435|0443"`
+numC3xxxDevice=`lspci -vnd 8086: | egrep -c "19e2|19e3"`
+numDh895xDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_NUMBER"`
+numDh895xDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_NUMBER_VM"`
+numC62xDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$C62X_DEVICE_NUMBER"`
+numD15xxDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_NUMBER"`
+numC3xxxDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$C3XXX_DEVICE_NUMBER"`
+MODPROBE_BLACKLIST_FILE="blacklist-qat-vfs.conf"
+
+# load vfio-pci
+$ECHO "Loading module vfio-pci"
+modprobe vfio-pci
+
+# qat-driver
+$ECHO "Installing driver in `\uname -r`"
+INSTALL_MOD_DIR=/lib/modules/`\uname -r`/updates/
+for ko in $KO_MODULES_LIST; do
+ base=${ko%/*};
+ file=${ko##*/};
+ mkdir -p $ROOT$INSTALL_MOD_DIR$base
+ $INSTALL $file $ROOT$INSTALL_MOD_DIR$base
+done
+
+# qat-adf-ctl
+if [ ! -d $ROOT$bindir ]; then
+ $MKDIR_P $ROOT$bindir;
+fi;
+$INSTALL -D -m 750 adf_ctl $ROOT$bindir/adf_ctl;
+
+# qat-service
+if [ ! -d $ROOT/lib/firmware/qat_fw_backup ]; then
+ $MKDIR_P $ROOT/lib/firmware/qat_fw_backup;
+fi;
+
+for bin in $BIN_LIST; do
+ if [ -e $ROOT/lib/firmware/$bin ]; then
+ mv $ROOT/lib/firmware/$bin $ROOT/lib/firmware/qat_fw_backup/$bin;
+ fi;
+ if [ -e $bin ]; then
+ $INSTALL -D -m 750 $bin $ROOT/lib/firmware/$bin;
+ fi;
+done;
+
+if [ ! -d $ROOT/etc/qat_conf_backup ]; then
+ $MKDIR_P $ROOT/etc/qat_conf_backup;
+fi;
+$MV $ROOT/etc/dh895xcc*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null;
+$MV $ROOT/etc/c6xx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null;
+$MV $ROOT/etc/d15xx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null;
+$MV $ROOT/etc/c3xxx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null;
+
+for ((dev=0; dev<$numDh895xDevicesP; dev++))
+do
+ $INSTALL -D -m 640 dh895xcc_dev0.conf $ROOT/etc/dh895xcc_dev$dev.conf;
+ for ((vf_dev = 0; vf_dev<$QAT_DH895XCC_NUM_VFS; vf_dev++))
+ do
+ vf_dev_num=$(($dev * $QAT_DH895XCC_NUM_VFS + $vf_dev));
+ $INSTALL -D -m 640 dh895xccvf_dev0.conf.vm $ROOT/etc/dh895xccvf_dev$vf_dev_num.conf;
+ done;
+done;
+
+for ((dev=0; dev<$numC62xDevicesP; dev++))
+do
+ $INSTALL -D -m 640 c6xx_dev$(($dev%3)).conf $ROOT/etc/c6xx_dev$dev.conf;
+ for ((vf_dev = 0; vf_dev<$QAT_DHC62X_NUM_VFS; vf_dev++))
+ do
+ vf_dev_num=$(($dev * $QAT_DHC62X_NUM_VFS + $vf_dev));
+ $INSTALL -D -m 640 c6xxvf_dev0.conf.vm $ROOT/etc/c6xxvf_dev$vf_dev_num.conf;
+ done;
+done;
+
+for ((dev=0; dev<$numD15xxDevicesP; dev++))
+do
+ $INSTALL -D -m 640 d15xx_dev$(($dev%3)).conf $ROOT/etc/d15xx_dev$dev.conf;
+ for ((vf_dev = 0; vf_dev<$QAT_DHD15XX_NUM_VFS; vf_dev++))
+ do
+ vf_dev_num=$(($dev * $QAT_DHD15XX_NUM_VFS + $vf_dev));
+ $INSTALL -D -m 640 d15xxvf_dev0.conf.vm $ROOT/etc/d15xxvf_dev$vf_dev_num.conf;
+ done;
+done;
+
+for ((dev=0; dev<$numC3xxxDevicesP; dev++))
+do
+ $INSTALL -D -m 640 c3xxx_dev0.conf $ROOT/etc/c3xxx_dev$dev.conf;
+ for ((vf_dev = 0; vf_dev<$QAT_DHC3XXX_NUM_VFS; vf_dev++))
+ do
+ vf_dev_num=$(($dev * $QAT_DHC3XXX_NUM_VFS + $vf_dev));
+ $INSTALL -D -m 640 c3xxxvf_dev0.conf.vm $ROOT/etc/c3xxxvf_dev$vf_dev_num.conf;
+ done;
+done;
+
+$ECHO "Creating startup and kill scripts";
+if [ ! -d $ROOT/etc/modprobe.d ]; then
+ $MKDIR_P $ROOT/etc/modprobe.d;
+fi;
+$INSTALL -D -m 750 qat_service $ROOT/etc/init.d/qat_service;
+$INSTALL -D -m 750 qat_service_vfs $ROOT/etc/init.d/qat_service_vfs;
+$INSTALL -D -m 750 qat $ROOT/etc/default/qat;
+if [ -e $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE ] ; then
+ $RM $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE;
+fi;
+
+if [ $numDh895xDevicesP != 0 ];then
+ $ECHO "blacklist qat_dh895xccvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE;
+fi;
+if [ $numC3xxxDevicesP != 0 ];then
+ $ECHO "blacklist qat_c3xxxvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE;
+fi;
+if [ $numC62xDevicesP != 0 ];then
+ $ECHO "blacklist qat_c62xvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE;
+fi;
+if [ $numD15xxDevicesP != 0 ];then
+ $ECHO "blacklist qat_d15xxvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE;
+fi;
+
+if [ ! -d $ROOT$libdir ]; then
+ $MKDIR_P $ROOT$libdir;
+fi;
+if [ ! -d $ROOT/etc/ld.so.conf.d ]; then
+ $MKDIR_P $ROOT/etc/ld.so.conf.d;
+fi;
+if [ ! -d $ROOT/lib/modules/`\uname -r`/kernel/drivers ]; then
+ $MKDIR_P $ROOT/lib/modules/`\uname -r`/kernel/drivers;
+fi;
+if [ ! -d $ROOT/etc/udev/rules.d ]; then
+ $MKDIR_P $ROOT/etc/udev/rules.d;
+fi;
+
+$ECHO "Copying libqat_s.so to $ROOT$libdir";
+$INSTALL -D -m 755 libqat_s.so $ROOT$libdir/libqat_s.so;
+$ECHO "Copying libusdm_drv_s.so to $ROOT$libdir";
+$INSTALL -D -m 755 libusdm_drv_s.so $ROOT$libdir/libusdm_drv_s.so;
+$ECHO $libdir > $ROOT/etc/ld.so.conf.d/qat.conf; ldconfig;
+
+$ECHO "Copying usdm module to system drivers";
+$INSTALL usdm_drv.ko "$ROOT/lib/modules/`\uname -r`/kernel/drivers";
+$INSTALL qat_api.ko "$ROOT/lib/modules/`\uname -r`/kernel/drivers";
+$ECHO "Creating udev rules";
+if [ ! -e $ROOT/etc/udev/rules.d/00-qat.rules ]; then
+ echo 'KERNEL=="qat_adf_ctl" MODE="0660" GROUP="qat"' > $ROOT/etc/udev/rules.d/00-qat.rules;
+ echo 'KERNEL=="qat_dev_processes" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules;
+ echo 'KERNEL=="usdm_drv" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules;
+ echo 'KERNEL=="uio*" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules;
+ echo 'KERNEL=="hugepages" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules;
+fi;
+$ECHO "Creating module.dep file for QAT released kernel object";
+$ECHO "This will take a few moments";
+depmod -a;
+if [ `lsmod | grep "usdm_drv" | wc -l` != "0" ]; then
+ $ECHO "rmmod usdm_drv";
+ rmmod usdm_drv;
+fi;
+if [ -e /sbin/chkconfig ] ; then
+ chkconfig --add qat_service;
+elif [ -e /usr/sbin/update-rc.d ]; then
+ $ECHO "update-rc.d qat_service defaults";
+ update-rc.d qat_service defaults;
+fi;
+
+$ECHO "Starting QAT service";
+/etc/init.d/qat_service shutdown;
+$SLEEP 3;
+/etc/init.d/qat_service start;
+
+# load kernel vf module for QAT device plugin
+numC62xDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$C62X_DEVICE_NUMBER_VM"`
+numD15xxDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_NUMBER_VM"`
+numC3xxxDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$C3XXX_DEVICE_NUMBER_VM"`
+if [ $numC62xDevicesV != 0 ];then
+ $ECHO "Loading qat_c62xvf";
+ modprobe qat_c62xvf
+fi
+if [ $numC3xxxDevicesV != 0 ];then
+ $ECHO "Loading qat_c3xxxvf";
+ modprobe qat_c3xxxvf
+fi
+if [ $numD15xxDevicesV != 0 ];then
+ $ECHO "Loading qat_d15xxvf";
+ modprobe qat_d15xxvf
+fi
+
--- /dev/null
+# Comment or remove next line to disable sriov
+SRIOV_ENABLE=1
+#LEGACY_LOADED=1
+#ENABLE_KAPI=1
--- /dev/null
+kind: Pod
+apiVersion: v1
+metadata:
+ name: dpdk2
+spec:
+ containers:
+ - name: dpdkcontainer
+ image: crypto-perf:devel
+ imagePullPolicy: IfNotPresent
+ command: [ "/bin/bash", "-c", "--" ]
+ args: [ "while true; do sleep 300000; done;" ]
+ volumeMounts:
+ - mountPath: /dev/hugepages
+ name: hugepage
+ resources:
+ requests:
+ cpu: "3"
+ memory: "1Gi"
+ qat.intel.com/generic: '2'
+ hugepages-2Mi: "1Gi"
+ limits:
+ cpu: "3"
+ memory: "1Gi"
+ qat.intel.com/generic: '2'
+ hugepages-2Mi: "1Gi"
+ securityContext:
+ capabilities:
+ add:
+ ["IPC_LOCK"]
+ volumes:
+ - name: hugepage
+ emptyDir:
+ medium: HugePages
--- /dev/null
+#!/bin/bash
+
+# usage: collect_qat_yaml.sh [target]
+
+set -ex
+
+if [ $# -ne 1 ] ; then
+ echo "Please input the target folder!"
+ exit 0
+fi
+
+VER="0.1"
+MKDIR_P="mkdir -p"
+target=$1
+temp=qat_yaml
+
+# copy to target
+$MKDIR_P $temp
+cp qat_plugin_default_configmap.yaml $temp/
+cp qat_plugin_privileges.yaml $temp/
+echo "#!/bin/bash" >> $temp/install.sh
+echo "cat qat_plugin_default_configmap.yaml | kubectl apply -f -" >> $temp/install.sh
+echo "cat qat_plugin_privileges.yaml | kubectl apply -f -" >> $temp/install.sh
+
+if [ ! -d $target/yaml ]; then
+ $MKDIR_P $target/yaml;
+fi;
+
+tar czvf $target/yaml/qat_yaml-$VER.tar.gz $temp/
+
+# clear
+rm -rf $temp
--- /dev/null
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: intel-qat-plugin-config
+data:
+ DPDK_DRIVER: "vfio-pci"
+ KERNEL_VF_DRIVERS: "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf"
+ MAX_NUM_DEVICES: "32"
+ DEBUG: "false"
--- /dev/null
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: intel-qat-plugin
+ labels:
+ app: intel-qat-plugin
+spec:
+ selector:
+ matchLabels:
+ app: intel-qat-plugin
+ template:
+ metadata:
+ labels:
+ app: intel-qat-plugin
+ spec:
+ containers:
+ - name: intel-qat-plugin
+ image: intel-qat-plugin:devel
+ env:
+ - name: DPDK_DRIVER
+ valueFrom:
+ configMapKeyRef:
+ name: intel-qat-plugin-config
+ key: DPDK_DRIVER
+ - name: KERNEL_VF_DRIVERS
+ valueFrom:
+ configMapKeyRef:
+ name: intel-qat-plugin-config
+ key: KERNEL_VF_DRIVERS
+ - name: MAX_NUM_DEVICES
+ valueFrom:
+ configMapKeyRef:
+ name: intel-qat-plugin-config
+ key: MAX_NUM_DEVICES
+ - name: DEBUG
+ valueFrom:
+ configMapKeyRef:
+ name: intel-qat-plugin-config
+ key: DEBUG
+ imagePullPolicy: IfNotPresent
+ command: ["/usr/bin/intel_qat_device_plugin", "-dpdk-driver", "$(DPDK_DRIVER)", "-kernel-vf-drivers", "$(KERNEL_VF_DRIVERS)", "-max-num-devices", "$(MAX_NUM_DEVICES)", "-debug", "$(DEBUG)"]
+ volumeMounts:
+ - name: pcidir
+ mountPath: /sys/bus/pci
+ - name: kubeletsockets
+ mountPath: /var/lib/kubelet/device-plugins
+ securityContext:
+ privileged: true
+ volumes:
+ - name: pcidir
+ hostPath:
+ path: /sys/bus/pci
+ - name: kubeletsockets
+ hostPath:
+ path: /var/lib/kubelet/device-plugins
+# Precondition: iavf.ko is pre-compiled and put in the same folder with install_iavf_driver.sh`
+# Intel Network Adapter Virtual Function Driver for Intel Ethernet Controller 700 series
+# https://downloadcenter.intel.com/download/24693/Intel-Network-Adapter-Virtual-Function-Driver-for-Intel-Ethernet-Controller-700-Series?product=82947
+
+# install sriov nic driver
+cd driver
+./install_iavf_driver.sh $ifname
+
+# install SRIOV device plugin
+cd yaml
+cat sriov-cni.yaml | kubectl apply -f -
+cat sriovdp-daemonset.yaml | kubectl apply -f -
+
+# test
+cd test
+cat sriov-nad.yaml | kubectl apply -f -
+cat sriov-eno2-pod.yaml | kubectl apply -f -
+
--- /dev/null
+#!/bin/bash
+
+# usage: collect_sriov_driver.sh [target]
+
+set -ex
+
+if [ $# -ne 1 ] ; then
+ echo "Please input the target folder!"
+ exit 0
+fi
+
+VER="0.1"
+MKDIR_P="mkdir -p"
+target=$1
+package=iavf-3.7.34
+
+# download driver source package
+if [ ! -e /tmp/$package.tar.gz ]; then
+ wget -P /tmp https://downloadmirror.intel.com/28943/eng/$package.tar.gz
+fi
+cp /tmp/$package.tar.gz .
+
+# compile
+tar xzvf $package.tar.gz
+pushd `pwd`
+cd $package/src
+make
+popd
+
+# copy to target
+$MKDIR_P sriov_driver
+cp $package/src/iavf.ko sriov_driver/
+cp install_iavf_drivers.sh sriov_driver/install.sh
+
+if [ ! -d $target/driver ]; then
+ $MKDIR_P $target/driver;
+fi;
+
+tar czvf $target/driver/sriov_driver-$VER.tar.gz sriov_driver/
+
+# clear
+rm -rf $package
+rm -rf sriov_driver
+rm $package.tar.gz
--- /dev/null
+#!/bin/bash
+
+function install_iavf_driver {
+ local ifname=$1
+
+ echo "Installing modules..."
+ echo "Installing i40evf blacklist file..."
+ mkdir -p "/etc/modprobe.d/"
+ echo "blacklist i40evf" > "/etc/modprobe.d/iavf-blacklist-i40evf.conf"
+
+ kver=`uname -a | awk '{print $3}'`
+ install_mod_dir=/lib/modules/$kver/updates/drivers/net/ethernet/intel/iavf/
+ echo "Installing driver in $install_mod_dir"
+ mkdir -p $install_mod_dir
+ cp iavf.ko $install_mod_dir
+
+ echo "Installing kernel module i40evf..."
+ depmod -a
+ modprobe i40evf
+ modprobe iavf
+
+ echo "Enabling VF on interface $ifname..."
+ echo "/sys/class/net/$ifname/device/sriov_numvfs"
+ echo '8' > /sys/class/net/$ifname/device/sriov_numvfs
+}
+
+function is_used {
+ local ifname=$1
+ route_info=`ip route show | grep $ifname`
+ if [ -z "$route_info" ]; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+function get_sriov_ifname {
+ for net_device in /sys/class/net/*/ ; do
+ if [ -e $net_device/device/sriov_numvfs ] ; then
+ ifname=$(basename $net_device)
+ is_used $ifname
+ if [ "$?" = "0" ]; then
+ echo $ifname
+ return
+ fi
+ fi
+ done
+ echo ''
+}
+
+if [ $# -ne 1 ] ; then
+ ifname=$(get_sriov_ifname)
+ if [ -z "$ifname" ]; then
+ echo "Cannot find Nic with SRIOV support."
+ else
+ install_iavf_driver $ifname
+ fi
+else
+ ifname=$1
+ if [ ! -e /sys/class/net/$ifname/device/sriov_numvfs ] ; then
+ echo "${ifname} is not a valid sriov interface"
+ else
+ install_iavf_driver $ifname
+ fi
+fi
--- /dev/null
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-case-01
+ annotations:
+ k8s.v1.cni.cncf.io/networks: sriov-eno2
+spec:
+ containers:
+ - name: test-pod
+ image: docker.io/centos/tools:latest
+ command:
+ - /sbin/init
+ resources:
+ requests:
+ intel.com/intel_sriov_700: '1'
+ limits:
+ intel.com/intel_sriov_700: '1'
+
--- /dev/null
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+ name: sriov-eno2
+ annotations:
+ k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_700
+spec:
+ config: '{
+ "type": "sriov",
+ "ipam": {
+ "type": "host-local",
+ "subnet": "10.56.206.0/24",
+ "routes": [
+ { "dst": "0.0.0.0/0" }
+ ],
+ "gateway": "10.56.206.1"
+ }
+ }'
--- /dev/null
+#!/bin/bash
+
+# usage: collect_sriov_yaml.sh [target]
+
+set -ex
+
+if [ $# -ne 1 ] ; then
+ echo "Please input the target folder!"
+ exit 0
+fi
+
+VER="0.1"
+MKDIR_P="mkdir -p"
+target=$1
+temp=sriov_yaml
+
+# copy to target
+$MKDIR_P $temp
+cp sriov-cni.yaml $temp/
+cp sriovdp-daemonset.yaml $temp/
+echo "#!/bin/bash" >> $temp/install.sh
+echo "cat sriov-cni.yaml | kubectl apply -f -" >> $temp/install.sh
+echo "cat sriovdp-daemonset.yaml | kubectl apply -f -" >> $temp/install.sh
+
+if [ ! -d $target/yaml ]; then
+ $MKDIR_P $target/yaml;
+fi;
+
+tar czvf $target/yaml/sriov_yaml-$VER.tar.gz $temp/
+
+# clear
+rm -rf $temp
--- /dev/null
+# SRIOV-CNI Release v1
+# Based on:
+# https://github.com/intel/sriov-cni/blob/master/images/sriov-cni-daemonset.yaml
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-sriov-cni-ds-amd64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: sriov-cni
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: sriov-cni
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ beta.kubernetes.io/arch: amd64
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ containers:
+ - name: kube-sriov-cni
+ image: nfvpe/sriov-cni
+ securityContext:
+ privileged: true
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50Mi"
+ limits:
+ cpu: "100m"
+ memory: "50Mi"
+ volumeMounts:
+ - name: cnibin
+ mountPath: /host/opt/cni/bin
+ volumes:
+ - name: cnibin
+ hostPath:
+ path: /opt/cni/bin
+
--- /dev/null
+# SRIOV device CNI plugin
+# Based on:
+# https://github.com/intel/sriov-network-device-plugin/blob/master/images/sriovdp-daemonset.yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sriovdp-config
+ namespace: kube-system
+data:
+ config.json: |
+ {
+ "resourceList": [{
+ "resourceName": "intel_sriov_700",
+ "selectors": {
+ "vendors": ["8086"]
+ }
+ }]
+ }
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: sriov-device-plugin
+ namespace: kube-system
+
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-sriov-device-plugin-amd64
+ namespace: kube-system
+ labels:
+ tier: node
+ app: sriovdp
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: sriovdp
+ spec:
+ hostNetwork: true
+ hostPID: true
+ nodeSelector:
+ beta.kubernetes.io/arch: amd64
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ serviceAccountName: sriov-device-plugin
+ containers:
+ - name: kube-sriovdp
+ image: nfvpe/sriov-device-plugin
+ args:
+ - --log-dir=sriovdp
+ - --log-level=10
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: devicesock
+ mountPath: /var/lib/kubelet/
+ readOnly: false
+ - name: log
+ mountPath: /var/log
+ - name: config-volume
+ mountPath: /etc/pcidp
+ volumes:
+ - name: devicesock
+ hostPath:
+ path: /var/lib/kubelet/
+ - name: log
+ hostPath:
+ path: /var/log
+ - name: config-volume
+ configMap:
+ name: sriovdp-config
+ items:
+ - key: config.json
+ path: config.json
--- /dev/null
+!/bin/bash
+
+# This script is called by cloud-init on worker nodes
+# What does this script do:
+# 1. Copy qat driver tarball and sriov tarball from share folder
+# 2. Extract the tarball and run install.sh to install the drivers
+
+# Need a variable named $SHARE_FOLDER to indicate the share folder location
+
+MODULES_LIST="qat_driver sriov_driver"
+VER="0.1"
+SHARE_FOLDER=${SHARE_FOLDER:-"package"}
+
+for module in $MODULES_LIST; do
+ filename=$module-$VER.tar.gz
+ if [ ! -e $filename ]; then
+ if [ ! -e $SHARE_FOLDER/$filename ]; then
+ echo "Cannot install module $module ..."
+ continue
+ else
+ cp $SHARE_FOLDER/$filename .
+ fi
+ fi
+
+ tar xvzf $filename
+ if [ -d $module ]; then
+ echo "Installing module $module ..."
+ pushd $module
+ bash ./install.sh
+ popd
+ rm -rf $module
+ fi
+done
+
Code Review / icn.git / commitdiff