From d093ee40c1659722c6547b49806f473c87f58aa2 Mon Sep 17 00:00:00 2001 From: hle2 Date: Sat, 3 Aug 2019 09:32:04 +0800 Subject: [PATCH] Add SRIOV and QAT device plugin deploy components Add deploy components include (1) SRIOV/QAT libraries (2) device plugin yaml for SRIOV/QAT device plugin. Change-Id: I1ae395d4d51d187d8fc5cb8f7a4d5b5e50ea8433 --- deploy/kud-plugin-addons/README.md | 20 ++ .../kud-plugin-addons/device-plugins/qat/README.md | 26 +++ .../qat/driver/collect_qat_driver.sh | 47 ++++ .../device-plugins/qat/driver/install_qat.sh | 245 +++++++++++++++++++++ .../device-plugins/qat/driver/qat | 4 + .../device-plugins/qat/test/test.yaml | 33 +++ .../device-plugins/qat/yaml/collect_qat_yaml.sh | 32 +++ .../qat/yaml/qat_plugin_default_configmap.yaml | 9 + .../qat/yaml/qat_plugin_privileges.yaml | 55 +++++ .../device-plugins/sriov/README.md | 18 ++ .../sriov/driver/collect_sriov_driver.sh | 44 ++++ .../sriov/driver/install_iavf_drivers.sh | 65 ++++++ .../device-plugins/sriov/test/sriov-eno2-pod.yaml | 18 ++ .../device-plugins/sriov/test/sriov-nad.yaml | 18 ++ .../sriov/yaml/collect_sriov_yaml.sh | 32 +++ .../device-plugins/sriov/yaml/sriov-cni.yaml | 46 ++++ .../sriov/yaml/sriovdp-daemonset.yaml | 81 +++++++ tools/cloud-configs.sh | 34 +++ 18 files changed, 827 insertions(+) create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/driver/collect_qat_driver.sh create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/driver/install_qat.sh create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/driver/qat create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/test/test.yaml create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/yaml/collect_qat_yaml.sh create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_default_configmap.yaml create mode 100644 deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_privileges.yaml create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/driver/collect_sriov_driver.sh create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-eno2-pod.yaml create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-nad.yaml create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/yaml/collect_sriov_yaml.sh create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriov-cni.yaml create mode 100644 deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriovdp-daemonset.yaml create mode 100644 tools/cloud-configs.sh diff --git a/deploy/kud-plugin-addons/README.md b/deploy/kud-plugin-addons/README.md index e69de29..0852c17 100644 --- a/deploy/kud-plugin-addons/README.md +++ b/deploy/kud-plugin-addons/README.md @@ -0,0 +1,20 @@ +kud-plugin-addons provides the scripts and yaml files to install kud-addons on working node which includes: device plugins, nfd, prometheus, rook etc. +1. device plugin: +(1) driver: include scripts to install driver on working node +collect_XX.sh: invoked by collect.sh in build machine to generate the driver installation package (e.g. qat_driver-0.1.tar.gz etc.), the package will be installed on /opt/icn/driver in icn infra-local-controller. BPA can install this package on cluster node, then install the driver with below commands: +$ tar xzvf qat_driver-0.1.tar.gz +$ cd qat_driver +$ bash install.sh + +(2) yaml: include the yaml file to install k8s device plugin +collect_XX.sh: invoked by collect.sh in build machine to generate the k8s device plugin installation package (e.g. qat_yaml-0.1.tar.gz etc.), the package will be installed on /opt/icn/yaml in icn infra-local-controller. BPA can install this package on cluster node, then install the device plugin with below commands(it assumed that k8s cluster had been installed by kud): +$ tar xzvf qat_yaml-0.1.tar.gz +$ cd qat_yaml +$ bash install.sh + +(3) test: include the test yaml file with sample workloads to use the device plugin + +2. nfd +3. prometheus +4. rook + diff --git a/deploy/kud-plugin-addons/device-plugins/qat/README.md b/deploy/kud-plugin-addons/device-plugins/qat/README.md index e69de29..9838c81 100644 --- a/deploy/kud-plugin-addons/device-plugins/qat/README.md +++ b/deploy/kud-plugin-addons/device-plugins/qat/README.md @@ -0,0 +1,26 @@ +# Precondition: +# (1) QAT device(e.g. 37c8) installed +# (2) Enable grub with "intel_iommu=on iommu=pt" +# (3) Driver install script is put at the same folder of QAT build target + +# QAT package: https://01.org/zh/intel-quick-assist-technology/downloads + +# install qat driver +cd driver +./install_qat.sh + +# install qat device plugin +# pre-pull local build docker image: intel-qat-plugin:devel +cd yaml +cat qat_plugin_default_configmap.yaml | kubectl apply -f - +cat qat_plugin_privileges.yaml | kubectl apply -f - + +# test +# pre-pull local build docker image: crypto-perf:devel +cd test +cat test.yaml | kubectl apply -f - +kubectl exec -it dpdk2 bash +./dpdk-test-crypto-perf -l 6-7 -w $QAT1 -- --ptest throughput\ + --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc\ + --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000\ + --burst-sz 32 --buffer-sz 64 diff --git a/deploy/kud-plugin-addons/device-plugins/qat/driver/collect_qat_driver.sh b/deploy/kud-plugin-addons/device-plugins/qat/driver/collect_qat_driver.sh new file mode 100644 index 0000000..b5f801a --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/driver/collect_qat_driver.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# usage: collect_qat_driver.sh [target] + +set -ex + +if [ $# -ne 1 ] ; then + echo "Please input the target folder!" + exit 0 +fi + +VER="0.1" +MKDIR_P="mkdir -p" +target=$1 +package=qat1.7.l.4.6.0-00025 + +# download driver source package +if [ ! -e /tmp/$package.tar.gz ]; then + wget -P /tmp https://01.org/sites/default/files/downloads/$package.tar.gz +fi +cp /tmp/$package.tar.gz . + +# compile +$MKDIR_P $package +tar xzvf $package.tar.gz -C $package +pushd `pwd` +cd $package +./configure --enable-icp-sriov=host +make +popd + +# copy to target +$MKDIR_P qat_driver +cp -r $package/build/* qat_driver/ +cp install_qat.sh qat_driver/install.sh +cp qat qat_driver/ + +if [ ! -d $target/driver ]; then + $MKDIR_P $target/driver; +fi; + +tar czvf $target/driver/qat_driver-$VER.tar.gz qat_driver/ + +# clear +rm -rf $package +rm -rf qat_driver +rm $package.tar.gz diff --git a/deploy/kud-plugin-addons/device-plugins/qat/driver/install_qat.sh b/deploy/kud-plugin-addons/device-plugins/qat/driver/install_qat.sh new file mode 100644 index 0000000..13c6f17 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/driver/install_qat.sh @@ -0,0 +1,245 @@ +#!/bin/bash + +# Precondition: +# QAT device installed, such as lspci | grep 37c8 +# Enable grub with "intel_iommu=on iommu=pt" + +ROOT= + +MV=mv +RM=rm +ECHO=echo +SLEEP=sleep +INSTALL="/usr/bin/install -c" +MKDIR_P="mkdir -p" +libdir=/usr/local/lib +bindir=/usr/local/bin + +am__append_1="drivers/crypto/qat/qat_dh895xcc/qat_dh895xcc.ko\ + drivers/crypto/qat/qat_dh895xccvf/qat_dh895xccvf.ko" + +am__append_2="qat_895xcc.bin qat_895xcc_mmp.bin" +am__append_3="dh895xcc_dev0.conf dh895xcc_dev1.conf dh895xccvf_dev0.conf.vm" + +# Kernel modules list +KO_MODULES_LIST="drivers/crypto/qat/qat_common/intel_qat.ko \ + drivers/crypto/qat/qat_c62x/qat_c62x.ko \ + drivers/crypto/qat/qat_c62xvf/qat_c62xvf.ko \ + drivers/crypto/qat/qat_d15xx/qat_d15xx.ko \ + drivers/crypto/qat/qat_d15xxvf/qat_d15xxvf.ko \ + drivers/crypto/qat/qat_c3xxx/qat_c3xxx.ko \ + drivers/crypto/qat/qat_c3xxxvf/qat_c3xxxvf.ko $am__append_1" + +# Firmwares list +BIN_LIST="qat_c3xxx.bin qat_c3xxx_mmp.bin qat_c62x.bin \ + qat_c62x_mmp.bin qat_mmp.bin qat_d15xx.bin qat_d15xx_mmp.bin \ + $am__append_2" +CONFIG_LIST="c3xxx_dev0.conf \ + c3xxxvf_dev0.conf.vm \ + c6xx_dev0.conf \ + c6xx_dev1.conf \ + c6xx_dev2.conf \ + c6xxvf_dev0.conf.vm \ + d15xx_dev0.conf \ + d15xxpf_dev0.conf \ + d15xxvf_dev0.conf.vm \ + $am__append_3" + +QAT_DH895XCC_NUM_VFS=32 +QAT_DHC62X_NUM_VFS=16 +QAT_DHD15XX_NUM_VFS=16 +QAT_DHC3XXX_NUM_VFS=16 + +# Device information variables +INTEL_VENDORID="8086" +DH895_DEVICE_NUMBER="0435" +DH895_DEVICE_NUMBER_VM="0443" +C62X_DEVICE_NUMBER="37c8" +C62X_DEVICE_NUMBER_VM="37c9" +D15XX_DEVICE_NUMBER="6f54" +D15XX_DEVICE_NUMBER_VM="6f55" +C3XXX_DEVICE_NUMBER="19e2" +C3XXX_DEVICE_NUMBER_VM="19e3" +numC62xDevice=`lspci -vnd 8086: | egrep -c "37c8|37c9"` +numD15xxDevice=`lspci -vnd 8086: | egrep -c "6f54|6f55"` +numDh895xDevice=`lspci -vnd 8086: | egrep -c "0435|0443"` +numC3xxxDevice=`lspci -vnd 8086: | egrep -c "19e2|19e3"` +numDh895xDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_NUMBER"` +numDh895xDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_NUMBER_VM"` +numC62xDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$C62X_DEVICE_NUMBER"` +numD15xxDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_NUMBER"` +numC3xxxDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$C3XXX_DEVICE_NUMBER"` +MODPROBE_BLACKLIST_FILE="blacklist-qat-vfs.conf" + +# load vfio-pci +$ECHO "Loading module vfio-pci" +modprobe vfio-pci + +# qat-driver +$ECHO "Installing driver in `\uname -r`" +INSTALL_MOD_DIR=/lib/modules/`\uname -r`/updates/ +for ko in $KO_MODULES_LIST; do + base=${ko%/*}; + file=${ko##*/}; + mkdir -p $ROOT$INSTALL_MOD_DIR$base + $INSTALL $file $ROOT$INSTALL_MOD_DIR$base +done + +# qat-adf-ctl +if [ ! -d $ROOT$bindir ]; then + $MKDIR_P $ROOT$bindir; +fi; +$INSTALL -D -m 750 adf_ctl $ROOT$bindir/adf_ctl; + +# qat-service +if [ ! -d $ROOT/lib/firmware/qat_fw_backup ]; then + $MKDIR_P $ROOT/lib/firmware/qat_fw_backup; +fi; + +for bin in $BIN_LIST; do + if [ -e $ROOT/lib/firmware/$bin ]; then + mv $ROOT/lib/firmware/$bin $ROOT/lib/firmware/qat_fw_backup/$bin; + fi; + if [ -e $bin ]; then + $INSTALL -D -m 750 $bin $ROOT/lib/firmware/$bin; + fi; +done; + +if [ ! -d $ROOT/etc/qat_conf_backup ]; then + $MKDIR_P $ROOT/etc/qat_conf_backup; +fi; +$MV $ROOT/etc/dh895xcc*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; +$MV $ROOT/etc/c6xx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; +$MV $ROOT/etc/d15xx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; +$MV $ROOT/etc/c3xxx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; + +for ((dev=0; dev<$numDh895xDevicesP; dev++)) +do + $INSTALL -D -m 640 dh895xcc_dev0.conf $ROOT/etc/dh895xcc_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DH895XCC_NUM_VFS; vf_dev++)) + do + vf_dev_num=$(($dev * $QAT_DH895XCC_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 dh895xccvf_dev0.conf.vm $ROOT/etc/dh895xccvf_dev$vf_dev_num.conf; + done; +done; + +for ((dev=0; dev<$numC62xDevicesP; dev++)) +do + $INSTALL -D -m 640 c6xx_dev$(($dev%3)).conf $ROOT/etc/c6xx_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DHC62X_NUM_VFS; vf_dev++)) + do + vf_dev_num=$(($dev * $QAT_DHC62X_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 c6xxvf_dev0.conf.vm $ROOT/etc/c6xxvf_dev$vf_dev_num.conf; + done; +done; + +for ((dev=0; dev<$numD15xxDevicesP; dev++)) +do + $INSTALL -D -m 640 d15xx_dev$(($dev%3)).conf $ROOT/etc/d15xx_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DHD15XX_NUM_VFS; vf_dev++)) + do + vf_dev_num=$(($dev * $QAT_DHD15XX_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 d15xxvf_dev0.conf.vm $ROOT/etc/d15xxvf_dev$vf_dev_num.conf; + done; +done; + +for ((dev=0; dev<$numC3xxxDevicesP; dev++)) +do + $INSTALL -D -m 640 c3xxx_dev0.conf $ROOT/etc/c3xxx_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DHC3XXX_NUM_VFS; vf_dev++)) + do + vf_dev_num=$(($dev * $QAT_DHC3XXX_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 c3xxxvf_dev0.conf.vm $ROOT/etc/c3xxxvf_dev$vf_dev_num.conf; + done; +done; + +$ECHO "Creating startup and kill scripts"; +if [ ! -d $ROOT/etc/modprobe.d ]; then + $MKDIR_P $ROOT/etc/modprobe.d; +fi; +$INSTALL -D -m 750 qat_service $ROOT/etc/init.d/qat_service; +$INSTALL -D -m 750 qat_service_vfs $ROOT/etc/init.d/qat_service_vfs; +$INSTALL -D -m 750 qat $ROOT/etc/default/qat; +if [ -e $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE ] ; then + $RM $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; + +if [ $numDh895xDevicesP != 0 ];then + $ECHO "blacklist qat_dh895xccvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; +if [ $numC3xxxDevicesP != 0 ];then + $ECHO "blacklist qat_c3xxxvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; +if [ $numC62xDevicesP != 0 ];then + $ECHO "blacklist qat_c62xvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; +if [ $numD15xxDevicesP != 0 ];then + $ECHO "blacklist qat_d15xxvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; + +if [ ! -d $ROOT$libdir ]; then + $MKDIR_P $ROOT$libdir; +fi; +if [ ! -d $ROOT/etc/ld.so.conf.d ]; then + $MKDIR_P $ROOT/etc/ld.so.conf.d; +fi; +if [ ! -d $ROOT/lib/modules/`\uname -r`/kernel/drivers ]; then + $MKDIR_P $ROOT/lib/modules/`\uname -r`/kernel/drivers; +fi; +if [ ! -d $ROOT/etc/udev/rules.d ]; then + $MKDIR_P $ROOT/etc/udev/rules.d; +fi; + +$ECHO "Copying libqat_s.so to $ROOT$libdir"; +$INSTALL -D -m 755 libqat_s.so $ROOT$libdir/libqat_s.so; +$ECHO "Copying libusdm_drv_s.so to $ROOT$libdir"; +$INSTALL -D -m 755 libusdm_drv_s.so $ROOT$libdir/libusdm_drv_s.so; +$ECHO $libdir > $ROOT/etc/ld.so.conf.d/qat.conf; ldconfig; + +$ECHO "Copying usdm module to system drivers"; +$INSTALL usdm_drv.ko "$ROOT/lib/modules/`\uname -r`/kernel/drivers"; +$INSTALL qat_api.ko "$ROOT/lib/modules/`\uname -r`/kernel/drivers"; +$ECHO "Creating udev rules"; +if [ ! -e $ROOT/etc/udev/rules.d/00-qat.rules ]; then + echo 'KERNEL=="qat_adf_ctl" MODE="0660" GROUP="qat"' > $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="qat_dev_processes" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="usdm_drv" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="uio*" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="hugepages" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; +fi; +$ECHO "Creating module.dep file for QAT released kernel object"; +$ECHO "This will take a few moments"; +depmod -a; +if [ `lsmod | grep "usdm_drv" | wc -l` != "0" ]; then + $ECHO "rmmod usdm_drv"; + rmmod usdm_drv; +fi; +if [ -e /sbin/chkconfig ] ; then + chkconfig --add qat_service; +elif [ -e /usr/sbin/update-rc.d ]; then + $ECHO "update-rc.d qat_service defaults"; + update-rc.d qat_service defaults; +fi; + +$ECHO "Starting QAT service"; +/etc/init.d/qat_service shutdown; +$SLEEP 3; +/etc/init.d/qat_service start; + +# load kernel vf module for QAT device plugin +numC62xDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$C62X_DEVICE_NUMBER_VM"` +numD15xxDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_NUMBER_VM"` +numC3xxxDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$C3XXX_DEVICE_NUMBER_VM"` +if [ $numC62xDevicesV != 0 ];then + $ECHO "Loading qat_c62xvf"; + modprobe qat_c62xvf +fi +if [ $numC3xxxDevicesV != 0 ];then + $ECHO "Loading qat_c3xxxvf"; + modprobe qat_c3xxxvf +fi +if [ $numD15xxDevicesV != 0 ];then + $ECHO "Loading qat_d15xxvf"; + modprobe qat_d15xxvf +fi + diff --git a/deploy/kud-plugin-addons/device-plugins/qat/driver/qat b/deploy/kud-plugin-addons/device-plugins/qat/driver/qat new file mode 100644 index 0000000..58513c7 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/driver/qat @@ -0,0 +1,4 @@ +# Comment or remove next line to disable sriov +SRIOV_ENABLE=1 +#LEGACY_LOADED=1 +#ENABLE_KAPI=1 diff --git a/deploy/kud-plugin-addons/device-plugins/qat/test/test.yaml b/deploy/kud-plugin-addons/device-plugins/qat/test/test.yaml new file mode 100644 index 0000000..0aa1126 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/test/test.yaml @@ -0,0 +1,33 @@ +kind: Pod +apiVersion: v1 +metadata: + name: dpdk2 +spec: + containers: + - name: dpdkcontainer + image: crypto-perf:devel + imagePullPolicy: IfNotPresent + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 300000; done;" ] + volumeMounts: + - mountPath: /dev/hugepages + name: hugepage + resources: + requests: + cpu: "3" + memory: "1Gi" + qat.intel.com/generic: '2' + hugepages-2Mi: "1Gi" + limits: + cpu: "3" + memory: "1Gi" + qat.intel.com/generic: '2' + hugepages-2Mi: "1Gi" + securityContext: + capabilities: + add: + ["IPC_LOCK"] + volumes: + - name: hugepage + emptyDir: + medium: HugePages diff --git a/deploy/kud-plugin-addons/device-plugins/qat/yaml/collect_qat_yaml.sh b/deploy/kud-plugin-addons/device-plugins/qat/yaml/collect_qat_yaml.sh new file mode 100644 index 0000000..6f4983b --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/yaml/collect_qat_yaml.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# usage: collect_qat_yaml.sh [target] + +set -ex + +if [ $# -ne 1 ] ; then + echo "Please input the target folder!" + exit 0 +fi + +VER="0.1" +MKDIR_P="mkdir -p" +target=$1 +temp=qat_yaml + +# copy to target +$MKDIR_P $temp +cp qat_plugin_default_configmap.yaml $temp/ +cp qat_plugin_privileges.yaml $temp/ +echo "#!/bin/bash" >> $temp/install.sh +echo "cat qat_plugin_default_configmap.yaml | kubectl apply -f -" >> $temp/install.sh +echo "cat qat_plugin_privileges.yaml | kubectl apply -f -" >> $temp/install.sh + +if [ ! -d $target/yaml ]; then + $MKDIR_P $target/yaml; +fi; + +tar czvf $target/yaml/qat_yaml-$VER.tar.gz $temp/ + +# clear +rm -rf $temp diff --git a/deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_default_configmap.yaml b/deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_default_configmap.yaml new file mode 100644 index 0000000..90d8289 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_default_configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: intel-qat-plugin-config +data: + DPDK_DRIVER: "vfio-pci" + KERNEL_VF_DRIVERS: "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf" + MAX_NUM_DEVICES: "32" + DEBUG: "false" diff --git a/deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_privileges.yaml b/deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_privileges.yaml new file mode 100644 index 0000000..d0032d8 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/qat/yaml/qat_plugin_privileges.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-qat-plugin + labels: + app: intel-qat-plugin +spec: + selector: + matchLabels: + app: intel-qat-plugin + template: + metadata: + labels: + app: intel-qat-plugin + spec: + containers: + - name: intel-qat-plugin + image: intel-qat-plugin:devel + env: + - name: DPDK_DRIVER + valueFrom: + configMapKeyRef: + name: intel-qat-plugin-config + key: DPDK_DRIVER + - name: KERNEL_VF_DRIVERS + valueFrom: + configMapKeyRef: + name: intel-qat-plugin-config + key: KERNEL_VF_DRIVERS + - name: MAX_NUM_DEVICES + valueFrom: + configMapKeyRef: + name: intel-qat-plugin-config + key: MAX_NUM_DEVICES + - name: DEBUG + valueFrom: + configMapKeyRef: + name: intel-qat-plugin-config + key: DEBUG + imagePullPolicy: IfNotPresent + command: ["/usr/bin/intel_qat_device_plugin", "-dpdk-driver", "$(DPDK_DRIVER)", "-kernel-vf-drivers", "$(KERNEL_VF_DRIVERS)", "-max-num-devices", "$(MAX_NUM_DEVICES)", "-debug", "$(DEBUG)"] + volumeMounts: + - name: pcidir + mountPath: /sys/bus/pci + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + securityContext: + privileged: true + volumes: + - name: pcidir + hostPath: + path: /sys/bus/pci + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/README.md b/deploy/kud-plugin-addons/device-plugins/sriov/README.md index e69de29..6506eaf 100644 --- a/deploy/kud-plugin-addons/device-plugins/sriov/README.md +++ b/deploy/kud-plugin-addons/device-plugins/sriov/README.md @@ -0,0 +1,18 @@ +# Precondition: iavf.ko is pre-compiled and put in the same folder with install_iavf_driver.sh` +# Intel Network Adapter Virtual Function Driver for Intel Ethernet Controller 700 series +# https://downloadcenter.intel.com/download/24693/Intel-Network-Adapter-Virtual-Function-Driver-for-Intel-Ethernet-Controller-700-Series?product=82947 + +# install sriov nic driver +cd driver +./install_iavf_driver.sh $ifname + +# install SRIOV device plugin +cd yaml +cat sriov-cni.yaml | kubectl apply -f - +cat sriovdp-daemonset.yaml | kubectl apply -f - + +# test +cd test +cat sriov-nad.yaml | kubectl apply -f - +cat sriov-eno2-pod.yaml | kubectl apply -f - + diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/driver/collect_sriov_driver.sh b/deploy/kud-plugin-addons/device-plugins/sriov/driver/collect_sriov_driver.sh new file mode 100644 index 0000000..6af4380 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/driver/collect_sriov_driver.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# usage: collect_sriov_driver.sh [target] + +set -ex + +if [ $# -ne 1 ] ; then + echo "Please input the target folder!" + exit 0 +fi + +VER="0.1" +MKDIR_P="mkdir -p" +target=$1 +package=iavf-3.7.34 + +# download driver source package +if [ ! -e /tmp/$package.tar.gz ]; then + wget -P /tmp https://downloadmirror.intel.com/28943/eng/$package.tar.gz +fi +cp /tmp/$package.tar.gz . + +# compile +tar xzvf $package.tar.gz +pushd `pwd` +cd $package/src +make +popd + +# copy to target +$MKDIR_P sriov_driver +cp $package/src/iavf.ko sriov_driver/ +cp install_iavf_drivers.sh sriov_driver/install.sh + +if [ ! -d $target/driver ]; then + $MKDIR_P $target/driver; +fi; + +tar czvf $target/driver/sriov_driver-$VER.tar.gz sriov_driver/ + +# clear +rm -rf $package +rm -rf sriov_driver +rm $package.tar.gz diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh b/deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh new file mode 100644 index 0000000..cdae5bb --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +function install_iavf_driver { + local ifname=$1 + + echo "Installing modules..." + echo "Installing i40evf blacklist file..." + mkdir -p "/etc/modprobe.d/" + echo "blacklist i40evf" > "/etc/modprobe.d/iavf-blacklist-i40evf.conf" + + kver=`uname -a | awk '{print $3}'` + install_mod_dir=/lib/modules/$kver/updates/drivers/net/ethernet/intel/iavf/ + echo "Installing driver in $install_mod_dir" + mkdir -p $install_mod_dir + cp iavf.ko $install_mod_dir + + echo "Installing kernel module i40evf..." + depmod -a + modprobe i40evf + modprobe iavf + + echo "Enabling VF on interface $ifname..." + echo "/sys/class/net/$ifname/device/sriov_numvfs" + echo '8' > /sys/class/net/$ifname/device/sriov_numvfs +} + +function is_used { + local ifname=$1 + route_info=`ip route show | grep $ifname` + if [ -z "$route_info" ]; then + return 0 + else + return 1 + fi +} + +function get_sriov_ifname { + for net_device in /sys/class/net/*/ ; do + if [ -e $net_device/device/sriov_numvfs ] ; then + ifname=$(basename $net_device) + is_used $ifname + if [ "$?" = "0" ]; then + echo $ifname + return + fi + fi + done + echo '' +} + +if [ $# -ne 1 ] ; then + ifname=$(get_sriov_ifname) + if [ -z "$ifname" ]; then + echo "Cannot find Nic with SRIOV support." + else + install_iavf_driver $ifname + fi +else + ifname=$1 + if [ ! -e /sys/class/net/$ifname/device/sriov_numvfs ] ; then + echo "${ifname} is not a valid sriov interface" + else + install_iavf_driver $ifname + fi +fi diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-eno2-pod.yaml b/deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-eno2-pod.yaml new file mode 100644 index 0000000..6ea187f --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-eno2-pod.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-case-01 + annotations: + k8s.v1.cni.cncf.io/networks: sriov-eno2 +spec: + containers: + - name: test-pod + image: docker.io/centos/tools:latest + command: + - /sbin/init + resources: + requests: + intel.com/intel_sriov_700: '1' + limits: + intel.com/intel_sriov_700: '1' + diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-nad.yaml b/deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-nad.yaml new file mode 100644 index 0000000..964938e --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/test/sriov-nad.yaml @@ -0,0 +1,18 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-eno2 + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_700 +spec: + config: '{ + "type": "sriov", + "ipam": { + "type": "host-local", + "subnet": "10.56.206.0/24", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "10.56.206.1" + } + }' diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/yaml/collect_sriov_yaml.sh b/deploy/kud-plugin-addons/device-plugins/sriov/yaml/collect_sriov_yaml.sh new file mode 100644 index 0000000..b9d150e --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/yaml/collect_sriov_yaml.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# usage: collect_sriov_yaml.sh [target] + +set -ex + +if [ $# -ne 1 ] ; then + echo "Please input the target folder!" + exit 0 +fi + +VER="0.1" +MKDIR_P="mkdir -p" +target=$1 +temp=sriov_yaml + +# copy to target +$MKDIR_P $temp +cp sriov-cni.yaml $temp/ +cp sriovdp-daemonset.yaml $temp/ +echo "#!/bin/bash" >> $temp/install.sh +echo "cat sriov-cni.yaml | kubectl apply -f -" >> $temp/install.sh +echo "cat sriovdp-daemonset.yaml | kubectl apply -f -" >> $temp/install.sh + +if [ ! -d $target/yaml ]; then + $MKDIR_P $target/yaml; +fi; + +tar czvf $target/yaml/sriov_yaml-$VER.tar.gz $temp/ + +# clear +rm -rf $temp diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriov-cni.yaml b/deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriov-cni.yaml new file mode 100644 index 0000000..f18c231 --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriov-cni.yaml @@ -0,0 +1,46 @@ +# SRIOV-CNI Release v1 +# Based on: +# https://github.com/intel/sriov-cni/blob/master/images/sriov-cni-daemonset.yaml +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-sriov-cni-ds-amd64 + namespace: kube-system + labels: + tier: node + app: sriov-cni +spec: + template: + metadata: + labels: + tier: node + app: sriov-cni + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: kube-sriov-cni + image: nfvpe/sriov-cni + securityContext: + privileged: true + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin + diff --git a/deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriovdp-daemonset.yaml b/deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriovdp-daemonset.yaml new file mode 100644 index 0000000..72743df --- /dev/null +++ b/deploy/kud-plugin-addons/device-plugins/sriov/yaml/sriovdp-daemonset.yaml @@ -0,0 +1,81 @@ +# SRIOV device CNI plugin +# Based on: +# https://github.com/intel/sriov-network-device-plugin/blob/master/images/sriovdp-daemonset.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: kube-system +data: + config.json: | + { + "resourceList": [{ + "resourceName": "intel_sriov_700", + "selectors": { + "vendors": ["8086"] + } + }] + } + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov-device-plugin + namespace: kube-system + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-sriov-device-plugin-amd64 + namespace: kube-system + labels: + tier: node + app: sriovdp +spec: + template: + metadata: + labels: + tier: node + app: sriovdp + spec: + hostNetwork: true + hostPID: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: sriov-device-plugin + containers: + - name: kube-sriovdp + image: nfvpe/sriov-device-plugin + args: + - --log-dir=sriovdp + - --log-level=10 + securityContext: + privileged: true + volumeMounts: + - name: devicesock + mountPath: /var/lib/kubelet/ + readOnly: false + - name: log + mountPath: /var/log + - name: config-volume + mountPath: /etc/pcidp + volumes: + - name: devicesock + hostPath: + path: /var/lib/kubelet/ + - name: log + hostPath: + path: /var/log + - name: config-volume + configMap: + name: sriovdp-config + items: + - key: config.json + path: config.json diff --git a/tools/cloud-configs.sh b/tools/cloud-configs.sh new file mode 100644 index 0000000..98ffcd4 --- /dev/null +++ b/tools/cloud-configs.sh @@ -0,0 +1,34 @@ +!/bin/bash + +# This script is called by cloud-init on worker nodes +# What does this script do: +# 1. Copy qat driver tarball and sriov tarball from share folder +# 2. Extract the tarball and run install.sh to install the drivers + +# Need a variable named $SHARE_FOLDER to indicate the share folder location + +MODULES_LIST="qat_driver sriov_driver" +VER="0.1" +SHARE_FOLDER=${SHARE_FOLDER:-"package"} + +for module in $MODULES_LIST; do + filename=$module-$VER.tar.gz + if [ ! -e $filename ]; then + if [ ! -e $SHARE_FOLDER/$filename ]; then + echo "Cannot install module $module ..." + continue + else + cp $SHARE_FOLDER/$filename . + fi + fi + + tar xvzf $filename + if [ -d $module ]; then + echo "Installing module $module ..." + pushd $module + bash ./install.sh + popd + rm -rf $module + fi +done + -- 2.16.6