From e4c1f2e47f191456c8d3b42c7acb41dc0c136195 Mon Sep 17 00:00:00 2001 From: davidplunkett Date: Thu, 23 May 2019 05:43:10 +0000 Subject: [PATCH] add vcpu_core_pin, gpu, and srio to nova Add support for vcpu_core_pin, gpu alias/pci whitelist, sriov alias settings to input file and nova template. Signed-off-by: davidplunkett Change-Id: I1e0d7e8f1d281cdfc9fbd572bf031d23d792053e --- dellgen10.yaml | 162 ++++++++++---------- hpgen10.yaml | 166 +++++++++++---------- .../charts/osh/openstack-compute-kit/nova.j2 | 25 +++- .../charts/osh/openstack-compute-kit/nova.j2 | 25 +++- 4 files changed, 214 insertions(+), 164 deletions(-) diff --git a/dellgen10.yaml b/dellgen10.yaml index 171020c..2e4caa0 100644 --- a/dellgen10.yaml +++ b/dellgen10.yaml @@ -115,81 +115,91 @@ dns: - 8.8.4.4 ingress_domain: dellgen10.akraino.org domain: lab.akraino.org -sriovnets: -- physical: sriovnet1 - interface: enp135s0f0 - vlan_start: 2001 - vlan_end: 3000 - whitelists: - - "address": "0000:87:02.0" - - "address": "0000:87:02.1" - - "address": "0000:87:03.2" - - "address": "0000:87:03.3" - - "address": "0000:87:03.4" - - "address": "0000:87:03.5" - - "address": "0000:87:03.6" - - "address": "0000:87:03.7" - - "address": "0000:87:04.0" - - "address": "0000:87:04.1" - - "address": "0000:87:04.2" - - "address": "0000:87:04.3" - - "address": "0000:87:02.2" - - "address": "0000:87:04.4" - - "address": "0000:87:04.5" - - "address": "0000:87:04.6" - - "address": "0000:87:04.7" - - "address": "0000:87:05.0" - - "address": "0000:87:05.1" - - "address": "0000:87:05.2" - - "address": "0000:87:05.3" - - "address": "0000:87:05.4" - - "address": "0000:87:05.5" - - "address": "0000:87:02.3" - - "address": "0000:87:05.6" - - "address": "0000:87:05.7" - - "address": "0000:87:02.4" - - "address": "0000:87:02.5" - - "address": "0000:87:02.6" - - "address": "0000:87:02.7" - - "address": "0000:87:03.0" - - "address": "0000:87:03.1" -- physical: sriovnet2 - interface: enp135s0f1 - vlan_start: 2001 - vlan_end: 3000 - whitelists: - - "address": "0000:87:0a.0" - - "address": "0000:87:0a.1" - - "address": "0000:87:0b.2" - - "address": "0000:87:0b.3" - - "address": "0000:87:0b.4" - - "address": "0000:87:0b.5" - - "address": "0000:87:0b.6" - - "address": "0000:87:0b.7" - - "address": "0000:87:0c.0" - - "address": "0000:87:0c.1" - - "address": "0000:87:0c.2" - - "address": "0000:87:0c.3" - - "address": "0000:87:0a.2" - - "address": "0000:87:0c.4" - - "address": "0000:87:0c.5" - - "address": "0000:87:0c.6" - - "address": "0000:87:0c.7" - - "address": "0000:87:0d.0" - - "address": "0000:87:0d.1" - - "address": "0000:87:0d.2" - - "address": "0000:87:0d.3" - - "address": "0000:87:0d.4" - - "address": "0000:87:0d.5" - - "address": "0000:87:0a.3" - - "address": "0000:87:0d.6" - - "address": "0000:87:0d.7" - - "address": "0000:87:0a.4" - - "address": "0000:87:0a.5" - - "address": "0000:87:0a.6" - - "address": "0000:87:0a.7" - - "address": "0000:87:0b.0" - - "address": "0000:87:0b.1" +gpu: + alias: + - name: "V100" + product_id: "1db4" + vendor_id: "10de" +sriov: + alias: + - name: "numa0" + product_id: "158b" + vendor_id: "8086" + nets: + - physical: sriovnet1 + interface: enp135s0f0 + vlan_start: 2001 + vlan_end: 3000 + whitelists: + - "address": "0000:87:02.0" + - "address": "0000:87:02.1" + - "address": "0000:87:03.2" + - "address": "0000:87:03.3" + - "address": "0000:87:03.4" + - "address": "0000:87:03.5" + - "address": "0000:87:03.6" + - "address": "0000:87:03.7" + - "address": "0000:87:04.0" + - "address": "0000:87:04.1" + - "address": "0000:87:04.2" + - "address": "0000:87:04.3" + - "address": "0000:87:02.2" + - "address": "0000:87:04.4" + - "address": "0000:87:04.5" + - "address": "0000:87:04.6" + - "address": "0000:87:04.7" + - "address": "0000:87:05.0" + - "address": "0000:87:05.1" + - "address": "0000:87:05.2" + - "address": "0000:87:05.3" + - "address": "0000:87:05.4" + - "address": "0000:87:05.5" + - "address": "0000:87:02.3" + - "address": "0000:87:05.6" + - "address": "0000:87:05.7" + - "address": "0000:87:02.4" + - "address": "0000:87:02.5" + - "address": "0000:87:02.6" + - "address": "0000:87:02.7" + - "address": "0000:87:03.0" + - "address": "0000:87:03.1" + - physical: sriovnet2 + interface: enp135s0f1 + vlan_start: 2001 + vlan_end: 3000 + whitelists: + - "address": "0000:87:0a.0" + - "address": "0000:87:0a.1" + - "address": "0000:87:0b.2" + - "address": "0000:87:0b.3" + - "address": "0000:87:0b.4" + - "address": "0000:87:0b.5" + - "address": "0000:87:0b.6" + - "address": "0000:87:0b.7" + - "address": "0000:87:0c.0" + - "address": "0000:87:0c.1" + - "address": "0000:87:0c.2" + - "address": "0000:87:0c.3" + - "address": "0000:87:0a.2" + - "address": "0000:87:0c.4" + - "address": "0000:87:0c.5" + - "address": "0000:87:0c.6" + - "address": "0000:87:0c.7" + - "address": "0000:87:0d.0" + - "address": "0000:87:0d.1" + - "address": "0000:87:0d.2" + - "address": "0000:87:0d.3" + - "address": "0000:87:0d.4" + - "address": "0000:87:0d.5" + - "address": "0000:87:0a.3" + - "address": "0000:87:0d.6" + - "address": "0000:87:0d.7" + - "address": "0000:87:0a.4" + - "address": "0000:87:0a.5" + - "address": "0000:87:0a.6" + - "address": "0000:87:0a.7" + - "address": "0000:87:0b.0" + - "address": "0000:87:0b.1" storage: osds: - data: /dev/sda @@ -243,6 +253,7 @@ masters: # ksn: 172.29.1.43 # neutron: 10.0.102.43 platform: + vcpu_pin_set: "4-21,26-43,48-65,72-87" kernel_params: hugepagesz: '1G' hugepages: 32 @@ -304,3 +315,4 @@ kubernetes: regional_server: ip: 135.16.101.85 ... + diff --git a/hpgen10.yaml b/hpgen10.yaml index 39c6d5e..8df3133 100644 --- a/hpgen10.yaml +++ b/hpgen10.yaml @@ -28,7 +28,7 @@ networks: - name: ens3f1 oob: vlan: 40 - interface: + interface: cidr: 192.168.41.0/24 routes: gateway: 192.168.41.1 @@ -61,7 +61,7 @@ networks: start: 172.31.1.2 end: 172.31.1.254 pxe: - vlan: + vlan: interface: eno1 cidr: 172.30.1.0/24 gateway: 172.30.1.1 @@ -109,81 +109,91 @@ dns: - 8.8.8.8 ingress_domain: hpgen10.akraino.org domain: lab.akraino.org -sriovnets: -- physical: sriovnet1 - interface: ens6f0 - vlan_start: 2001 - vlan_end: 3000 - whitelists: - - "address": "0000:af:02.0" - - "address": "0000:af:02.1" - - "address": "0000:af:02.2" - - "address": "0000:af:02.3" - - "address": "0000:af:02.4" - - "address": "0000:af:02.5" - - "address": "0000:af:02.6" - - "address": "0000:af:02.7" - - "address": "0000:af:03.0" - - "address": "0000:af:03.1" - - "address": "0000:af:03.2" - - "address": "0000:af:03.3" - - "address": "0000:af:03.4" - - "address": "0000:af:03.5" - - "address": "0000:af:03.6" - - "address": "0000:af:03.7" - - "address": "0000:af:04.0" - - "address": "0000:af:04.1" - - "address": "0000:af:04.2" - - "address": "0000:af:04.3" - - "address": "0000:af:04.4" - - "address": "0000:af:04.5" - - "address": "0000:af:04.6" - - "address": "0000:af:04.7" - - "address": "0000:af:05.0" - - "address": "0000:af:05.1" - - "address": "0000:af:05.2" - - "address": "0000:af:05.3" - - "address": "0000:af:05.4" - - "address": "0000:af:05.5" - - "address": "0000:af:05.6" - - "address": "0000:af:05.7" -- physical: sriovnet2 - interface: ens6f1 - vlan_start: 2001 - vlan_end: 3000 - whitelists: - - "address": "0000:af:0a.0" - - "address": "0000:af:0a.1" - - "address": "0000:af:0a.2" - - "address": "0000:af:0a.3" - - "address": "0000:af:0a.4" - - "address": "0000:af:0a.5" - - "address": "0000:af:0a.6" - - "address": "0000:af:0a.7" - - "address": "0000:af:0b.0" - - "address": "0000:af:0b.1" - - "address": "0000:af:0b.2" - - "address": "0000:af:0b.3" - - "address": "0000:af:0b.4" - - "address": "0000:af:0b.5" - - "address": "0000:af:0b.6" - - "address": "0000:af:0b.7" - - "address": "0000:af:0c.0" - - "address": "0000:af:0c.1" - - "address": "0000:af:0c.2" - - "address": "0000:af:0c.3" - - "address": "0000:af:0c.4" - - "address": "0000:af:0c.5" - - "address": "0000:af:0c.6" - - "address": "0000:af:0c.7" - - "address": "0000:af:0d.0" - - "address": "0000:af:0d.1" - - "address": "0000:af:0d.2" - - "address": "0000:af:0d.3" - - "address": "0000:af:0d.4" - - "address": "0000:af:0d.5" - - "address": "0000:af:0d.6" - - "address": "0000:af:0d.7" +gpu: + alias: + - name: "V100" + product_id: "1db4" + vendor_id: "10de" +sriov: + alias: + - name: "numa0" + product_id: "158b" + vendor_id: "8086" + nets: + - physical: sriovnet1 + interface: ens6f0 + vlan_start: 2001 + vlan_end: 3000 + whitelists: + - "address": "0000:af:02.0" + - "address": "0000:af:02.1" + - "address": "0000:af:02.2" + - "address": "0000:af:02.3" + - "address": "0000:af:02.4" + - "address": "0000:af:02.5" + - "address": "0000:af:02.6" + - "address": "0000:af:02.7" + - "address": "0000:af:03.0" + - "address": "0000:af:03.1" + - "address": "0000:af:03.2" + - "address": "0000:af:03.3" + - "address": "0000:af:03.4" + - "address": "0000:af:03.5" + - "address": "0000:af:03.6" + - "address": "0000:af:03.7" + - "address": "0000:af:04.0" + - "address": "0000:af:04.1" + - "address": "0000:af:04.2" + - "address": "0000:af:04.3" + - "address": "0000:af:04.4" + - "address": "0000:af:04.5" + - "address": "0000:af:04.6" + - "address": "0000:af:04.7" + - "address": "0000:af:05.0" + - "address": "0000:af:05.1" + - "address": "0000:af:05.2" + - "address": "0000:af:05.3" + - "address": "0000:af:05.4" + - "address": "0000:af:05.5" + - "address": "0000:af:05.6" + - "address": "0000:af:05.7" + - physical: sriovnet2 + interface: ens6f1 + vlan_start: 2001 + vlan_end: 3000 + whitelists: + - "address": "0000:af:0a.0" + - "address": "0000:af:0a.1" + - "address": "0000:af:0a.2" + - "address": "0000:af:0a.3" + - "address": "0000:af:0a.4" + - "address": "0000:af:0a.5" + - "address": "0000:af:0a.6" + - "address": "0000:af:0a.7" + - "address": "0000:af:0b.0" + - "address": "0000:af:0b.1" + - "address": "0000:af:0b.2" + - "address": "0000:af:0b.3" + - "address": "0000:af:0b.4" + - "address": "0000:af:0b.5" + - "address": "0000:af:0b.6" + - "address": "0000:af:0b.7" + - "address": "0000:af:0c.0" + - "address": "0000:af:0c.1" + - "address": "0000:af:0c.2" + - "address": "0000:af:0c.3" + - "address": "0000:af:0c.4" + - "address": "0000:af:0c.5" + - "address": "0000:af:0c.6" + - "address": "0000:af:0c.7" + - "address": "0000:af:0d.0" + - "address": "0000:af:0d.1" + - "address": "0000:af:0d.2" + - "address": "0000:af:0d.3" + - "address": "0000:af:0d.4" + - "address": "0000:af:0d.5" + - "address": "0000:af:0d.6" + - "address": "0000:af:0d.7" storage: osds: - data: /dev/sdb @@ -242,6 +252,7 @@ masters: # ksn: 172.29.1.33 # neutron: 10.0.101.33 platform: + vcpu_pin_set: "4-21,26-43,48-65,72-87" kernel_params: hugepagesz: '1G' hugepages: 32 @@ -303,3 +314,4 @@ kubernetes: regional_server: ip: 135.16.101.85 ... + diff --git a/site_type/ovsdpdk/templates/software/charts/osh/openstack-compute-kit/nova.j2 b/site_type/ovsdpdk/templates/software/charts/osh/openstack-compute-kit/nova.j2 index 3c44053..5b19f2b 100644 --- a/site_type/ovsdpdk/templates/software/charts/osh/openstack-compute-kit/nova.j2 +++ b/site_type/ovsdpdk/templates/software/charts/osh/openstack-compute-kit/nova.j2 @@ -43,15 +43,28 @@ data: libvirt: virt_type: kvm DEFAULT: - vcpu_pin_set: "4-21,26-43,48-65,72-87" + vcpu_pin_set: {% if 'platform' in yaml and 'vcpu_pin_set' in yaml.platform %}{{yaml.platform.vcpu_pin_set}} +{% else %}"4-21,26-43,48-65,72-87" +{% endif %} vif_plugging_is_fatal: False vif_plugging_timeout: 30 pci: -# alias: '{ "vendor_id":"10de", "product_id":"1db4", "name":"V100", "device_type":"type-PCI" }' -# passthrough_whitelist: '{"vendor_id": "10de", "product_id": "1db4"}' - alias: '{"name": "numa0", "capability_type": "pci", "product_id": "158b", "vendor_id": "8086", "device_type": "type-PCI", "numa_policy": "required"}`' -{% for sriovnet in yaml.sriovnets %} +{% if 'gpu' in yaml and 'alias' in yaml.gpu %} +{% for alias in yaml.gpu.alias %} + alias: '{ "name":"{{alias.name}}", "vendor_id":"{{alias.vendor_id}}", "product_id":"{{alias.product_id}}", "device_type":"type-PCI" }' + passthrough_whitelist: '{"vendor_id": "{{alias.vendor_id}}", "product_id": "{{alias.product_id}}"}' +{% endfor %} +{% endif %} +{% if 'sriov' in yaml and 'alias' in yaml.sriov %} +{% for alias in yaml.sriov.alias %} + alias: '{"name": "{{alias.name}}", "vendor_id": "{{alias.vendor_id}}", "product_id": "{{alias.product_id}}", "capability_type": "pci", "device_type": "type-PCI", "numa_policy": "required"}`' +{% endfor %} +{% endif %} +{% if 'sriov' in yaml and 'nets' in yaml.sriov %} +{% for sriovnet in yaml.sriov.nets %} passthrough_whitelist: | - [{% for vf in sriovnet.whitelists -%}{"address":"{{vf["address"]}}","physical_network":"{{sriovnet.physical}}"}{% if loop.index < sriovnet.whitelists|length %},{% endif %}{% endfor %}] + [{% for vf in sriovnet.whitelists -%}{"address":"{{vf["address"]}}","physical_network":"{{sriovnet.physical}}"}{{',' if not loop.last else ''}}{% endfor %}] {% endfor %} +{% endif %} ... + diff --git a/site_type/sriov/templates/software/charts/osh/openstack-compute-kit/nova.j2 b/site_type/sriov/templates/software/charts/osh/openstack-compute-kit/nova.j2 index 3c44053..5b19f2b 100644 --- a/site_type/sriov/templates/software/charts/osh/openstack-compute-kit/nova.j2 +++ b/site_type/sriov/templates/software/charts/osh/openstack-compute-kit/nova.j2 @@ -43,15 +43,28 @@ data: libvirt: virt_type: kvm DEFAULT: - vcpu_pin_set: "4-21,26-43,48-65,72-87" + vcpu_pin_set: {% if 'platform' in yaml and 'vcpu_pin_set' in yaml.platform %}{{yaml.platform.vcpu_pin_set}} +{% else %}"4-21,26-43,48-65,72-87" +{% endif %} vif_plugging_is_fatal: False vif_plugging_timeout: 30 pci: -# alias: '{ "vendor_id":"10de", "product_id":"1db4", "name":"V100", "device_type":"type-PCI" }' -# passthrough_whitelist: '{"vendor_id": "10de", "product_id": "1db4"}' - alias: '{"name": "numa0", "capability_type": "pci", "product_id": "158b", "vendor_id": "8086", "device_type": "type-PCI", "numa_policy": "required"}`' -{% for sriovnet in yaml.sriovnets %} +{% if 'gpu' in yaml and 'alias' in yaml.gpu %} +{% for alias in yaml.gpu.alias %} + alias: '{ "name":"{{alias.name}}", "vendor_id":"{{alias.vendor_id}}", "product_id":"{{alias.product_id}}", "device_type":"type-PCI" }' + passthrough_whitelist: '{"vendor_id": "{{alias.vendor_id}}", "product_id": "{{alias.product_id}}"}' +{% endfor %} +{% endif %} +{% if 'sriov' in yaml and 'alias' in yaml.sriov %} +{% for alias in yaml.sriov.alias %} + alias: '{"name": "{{alias.name}}", "vendor_id": "{{alias.vendor_id}}", "product_id": "{{alias.product_id}}", "capability_type": "pci", "device_type": "type-PCI", "numa_policy": "required"}`' +{% endfor %} +{% endif %} +{% if 'sriov' in yaml and 'nets' in yaml.sriov %} +{% for sriovnet in yaml.sriov.nets %} passthrough_whitelist: | - [{% for vf in sriovnet.whitelists -%}{"address":"{{vf["address"]}}","physical_network":"{{sriovnet.physical}}"}{% if loop.index < sriovnet.whitelists|length %},{% endif %}{% endfor %}] + [{% for vf in sriovnet.whitelists -%}{"address":"{{vf["address"]}}","physical_network":"{{sriovnet.physical}}"}{{',' if not loop.last else ''}}{% endfor %}] {% endfor %} +{% endif %} ... + -- 2.16.6