From 576b647dd9883ec1fb66c4fe1c72a9aa20dea713 Mon Sep 17 00:00:00 2001 From: Krisztian Lengyel Date: Tue, 16 Jul 2019 15:06:31 +0200 Subject: [PATCH] Upgrade DANM to version 4.0 - This upgrade introduces a new component named `danm_webhook`. - Previously statically templated DANM CRDs now also come form the repo in build time. - Use default tenant network on infra components Change-Id: I923b5e7aab7841006567681b071ddb0d292209cc Signed-off-by: Krisztian Lengyel --- SPECS/infra-charts.spec | 2 +- SPECS/instantiate.spec | 3 +- ansible/filter_plugins/config_helpers.py | 33 ++++++++++-- .../roles/install_caas_infra/defaults/main.yaml | 1 + .../roles/pre_install_caas_infra/meta/main.yaml | 31 +++++++++++ .../roles/pre_install_caas_infra/tasks/main.yaml | 4 +- ...epare_cpupooler.yaml => prepare_ca_bundle.yaml} | 2 +- cm_config/caas.yaml | 5 +- .../templates/cpu-deviceplugin-daemonset.yaml | 7 --- infra-charts/templates/cpu-setter-daemonset.yaml | 7 --- .../custom-metrics-apiserver-deployment.yaml | 7 --- infra-charts/templates/danm-webhook-config.yaml | 63 ++++++++++++++++++++++ infra-charts/templates/danm-webhook-ds.yaml | 52 ++++++++++++++++++ infra-charts/templates/danm-webhook-service.yaml | 32 +++++++++++ .../templates/elasticsearch-data-statefulset.yaml | 7 --- .../elasticsearch-master-statefulset.yaml | 7 --- infra-charts/templates/fluentd-daemonset.yaml | 6 --- .../templates/metrics-server-deployment.yaml | 7 --- infra-charts/templates/prometheus-dep.yaml | 6 --- infra-charts/templates/sriovdp-ds.yaml | 7 --- infra-charts/templates/svcwatcher.yaml | 7 --- .../templates/webhook-conf-updater-job.yaml | 7 --- infra-charts/templates/webhook-deployment.yaml | 7 --- infra-charts/values.yaml.j2 | 9 +++- 24 files changed, 226 insertions(+), 93 deletions(-) rename ansible/roles/pre_install_caas_infra/tasks/{prepare_cpupooler.yaml => prepare_ca_bundle.yaml} (96%) create mode 100644 infra-charts/templates/danm-webhook-config.yaml create mode 100644 infra-charts/templates/danm-webhook-ds.yaml create mode 100644 infra-charts/templates/danm-webhook-service.yaml diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec index c7d329e..a01d9af 100644 --- a/SPECS/infra-charts.spec +++ b/SPECS/infra-charts.spec @@ -15,7 +15,7 @@ %define COMPONENT infra-charts %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 1.0.0 -%define RPM_MINOR_VERSION 16 +%define RPM_MINOR_VERSION 17 Name: %{RPM_NAME} Version: %{RPM_MAJOR_VERSION} diff --git a/SPECS/instantiate.spec b/SPECS/instantiate.spec index 5ab7542..4465118 100644 --- a/SPECS/instantiate.spec +++ b/SPECS/instantiate.spec @@ -15,7 +15,7 @@ %define COMPONENT instantiate %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 1.0.0 -%define RPM_MINOR_VERSION 9 +%define RPM_MINOR_VERSION 10 Name: %{RPM_NAME} Version: %{RPM_MAJOR_VERSION} @@ -81,6 +81,7 @@ sed -ri '/^manifests_directory/{s|:.*|: %{_caas_manifest_path}|}' % sed -ri '/^rbac_manifests_directory/{s|:.*|: %{_caas_rbac_manifests_path}|}' %{buildroot}/%{_cm_caas_config_file} sed -ri '/^caas_chart_path/{s|:.*|: %{_caas_chart_path}|}' %{buildroot}/%{_cm_caas_config_file} sed -ri '/^libexec_dir/{s|:.*|: %{_caas_libexec_path}|}' %{buildroot}/%{_cm_caas_config_file} +sed -ri '/^danm_crd_dir/{s|:.*|: %{_caas_danm_crd_path}|}' %{buildroot}/%{_cm_caas_config_file} %files %{_playbooks_path}/* diff --git a/ansible/filter_plugins/config_helpers.py b/ansible/filter_plugins/config_helpers.py index 9ef5601..e21726a 100755 --- a/ansible/filter_plugins/config_helpers.py +++ b/ansible/filter_plugins/config_helpers.py @@ -14,20 +14,39 @@ # limitations under the License. from ansible.errors import AnsibleError +import itertools class FilterModule(object): def filters(self): return { + 'extract_sriov_provider_network_interfaces': extract_sriov_provider_network_interfaces, + 'filter_network_profiles_by_type': filter_network_profiles_by_type, + 'filter_provider_networks_by_type': filter_provider_networks_by_type, 'get_kube_options': get_kube_options, 'get_mapped_key': get_mapped_key, + 'get_provider_networks': get_provider_networks, } -def get_kube_options(options): - if not isinstance(options, dict): - raise AnsibleError("Invalid type {}. Options must be dictionary!".format(type(options))) +def extract_sriov_provider_network_interfaces(sriov_networks): + return list(itertools.chain.from_iterable( + [network.get('interfaces', []) + for network in sriov_networks.itervalues()])) + + +def filter_network_profiles_by_type(profiles, key, type): + return {name: profile for name, profile in profiles.iteritems() + if key in profile and filter((lambda x: x.get('type', "") == type), profile[key].itervalues())} + + +def filter_provider_networks_by_type(profile, type): + return {name: network for name, network in profile.iteritems() + if network.get('type', "") == type} + +def get_kube_options(options): + _validate_dict(options) option_template = "{}={}" formated_options = [option_template.format(option, str(value)) for option, value in options.iteritems()] @@ -42,6 +61,12 @@ def get_mapped_key(mapping, search_key, key_name): return key +def get_provider_networks(network_interfaces): + return list(itertools.chain.from_iterable( + [interface.get('provider_networks', []) + for interface in network_interfaces.itervalues()])) + + def _validate_dict(value): if not isinstance(value, dict): - raise AnsibleError("Invalid type {}. Options must be dictionary!".format(type(options))) + raise AnsibleError("Invalid type {}. Options must be dictionary!".format(type(value))) diff --git a/ansible/roles/install_caas_infra/defaults/main.yaml b/ansible/roles/install_caas_infra/defaults/main.yaml index 83953cb..d8af266 100644 --- a/ansible/roles/install_caas_infra/defaults/main.yaml +++ b/ansible/roles/install_caas_infra/defaults/main.yaml @@ -23,5 +23,6 @@ infra_chart_images: - svcwatcher - netwatcher - fluentd + - danm-webhook infra_chart_separated_values: - fluentd diff --git a/ansible/roles/pre_install_caas_infra/meta/main.yaml b/ansible/roles/pre_install_caas_infra/meta/main.yaml index 842164f..2fd4d95 100644 --- a/ansible/roles/pre_install_caas_infra/meta/main.yaml +++ b/ansible/roles/pre_install_caas_infra/meta/main.yaml @@ -14,6 +14,22 @@ # limitations under the License. dependencies: + - role: creategroup + _name: danm + _gid: "{{ caas.uid.danm }}" + become: true + become_user: "root" + + - role: createuser + _name: danm + _group: danm + _groups: '' + _shell: /sbin/nologin + _home: / + _uid: "{{ caas.uid.danm }}" + become: true + become_user: "root" + - role: cert instance: "cpuwebhook" cert_name: "cpuwebhook.crt" @@ -26,6 +42,21 @@ dependencies: become: true become_user: "root" + - role: cert + instance: "danm-webhook" + cert_name: "danm_webhook.crt" + key_name: "danm_webhook.key" + cert_path: /etc/kubernetes/ssl + common_name: "danm-webhook" + add_users: + - danm + alt_names: + dns: + - "danm-webhook-svc.kube-system.svc.{{ caas.dns_domain }}" + - "danm-webhook-svc.kube-system.svc" + become: true + become_user: "root" + - role: cert instance: "prometheus" cert_name: "prometheus.crt" diff --git a/ansible/roles/pre_install_caas_infra/tasks/main.yaml b/ansible/roles/pre_install_caas_infra/tasks/main.yaml index 4eef782..e35a01e 100644 --- a/ansible/roles/pre_install_caas_infra/tasks/main.yaml +++ b/ansible/roles/pre_install_caas_infra/tasks/main.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. - - name: prepare cpupooler - import_tasks: prepare_cpupooler.yaml + - name: prepare ca bundle + import_tasks: prepare_ca_bundle.yaml - name: prepare metrics import_tasks: prepare_metrics.yaml diff --git a/ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml b/ansible/roles/pre_install_caas_infra/tasks/prepare_ca_bundle.yaml similarity index 96% rename from ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml rename to ansible/roles/pre_install_caas_infra/tasks/prepare_ca_bundle.yaml index 5bfac33..af9d665 100644 --- a/ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml +++ b/ansible/roles/pre_install_caas_infra/tasks/prepare_ca_bundle.yaml @@ -19,7 +19,7 @@ become_user: "root" no_log: true -- name: set cpu_ca_bundle +- name: set ca_bundle set_fact: webhooks_ca: "{{ temp_webhook_ca_bundle.stdout }}" no_log: true diff --git a/cm_config/caas.yaml b/cm_config/caas.yaml index 414d529..521e37a 100644 --- a/cm_config/caas.yaml +++ b/cm_config/caas.yaml @@ -68,7 +68,7 @@ ldap_slave_url: infraldap-server-slave.kube-system.svc.{{ dns_domain }} apiserver_secure_port: 6443 apiserver_svc_port: 443 default_partition_size: 16 -danmnet_default_name: flannel +danmnet_default_name: default danmnet_default_namespace: kube-system danmnet_default_network_id: flannel danmnet_default_network_type: flannel @@ -117,3 +117,6 @@ swift_credential: user: admin tenant: admin password: willbechanged +danm_crd_dir: "{{ caas_base_directory }}/crds" +danm_webhook_port: 443 +danm_webhook_target_port: 6443 diff --git a/infra-charts/templates/cpu-deviceplugin-daemonset.yaml b/infra-charts/templates/cpu-deviceplugin-daemonset.yaml index b939620..ef8a7aa 100644 --- a/infra-charts/templates/cpu-deviceplugin-daemonset.yaml +++ b/infra-charts/templates/cpu-deviceplugin-daemonset.yaml @@ -28,13 +28,6 @@ spec: {{ .Values.kubernetes_component_label }}: cpu-device-plugin template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] labels: {{ .Values.kubernetes_component_label }}: cpu-device-plugin spec: diff --git a/infra-charts/templates/cpu-setter-daemonset.yaml b/infra-charts/templates/cpu-setter-daemonset.yaml index 1a5ccd2..07d822d 100644 --- a/infra-charts/templates/cpu-setter-daemonset.yaml +++ b/infra-charts/templates/cpu-setter-daemonset.yaml @@ -28,13 +28,6 @@ spec: {{ .Values.kubernetes_component_label }}: cpu-setter template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] labels: {{ .Values.kubernetes_component_label }}: cpu-setter spec: diff --git a/infra-charts/templates/custom-metrics-apiserver-deployment.yaml b/infra-charts/templates/custom-metrics-apiserver-deployment.yaml index d948503..92607c9 100755 --- a/infra-charts/templates/custom-metrics-apiserver-deployment.yaml +++ b/infra-charts/templates/custom-metrics-apiserver-deployment.yaml @@ -32,13 +32,6 @@ spec: labels: {{ .Values.kubernetes_component_label }}: custom-metrics-apiserver name: custom-metrics-apiserver - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] spec: priorityClassName: "system-cluster-critical" nodeSelector: diff --git a/infra-charts/templates/danm-webhook-config.yaml b/infra-charts/templates/danm-webhook-config.yaml new file mode 100644 index 0000000..e738e98 --- /dev/null +++ b/infra-charts/templates/danm-webhook-config.yaml @@ -0,0 +1,63 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.danm_webhook.required }} +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: danm-webhook-config + namespace: kube-system +webhooks: + - name: danm-netvalidation.nokia.k8s.io + clientConfig: + service: + name: danm-webhook-svc + namespace: kube-system + path: "/netvalidation" + caBundle: "{{ .Values.danm_webhook.webhook_ca_bundle }}" + rules: + - operations: ["CREATE","UPDATE"] + apiGroups: ["danm.k8s.io"] + apiVersions: ["v1"] + resources: ["clusternetworks","tenantnetworks"] + failurePolicy: Fail + - name: danm-configvalidation.nokia.k8s.io + clientConfig: + service: + name: danm-webhook-svc + namespace: kube-system + path: "/confvalidation" + caBundle: "{{ .Values.danm_webhook.webhook_ca_bundle }}" + rules: + - operations: ["CREATE","UPDATE"] + apiGroups: ["danm.k8s.io"] + apiVersions: ["v1"] + resources: ["tenantconfigs"] + failurePolicy: Fail + - name: danm-netdeletion.nokia.k8s.io + clientConfig: + service: + name: danm-webhook-svc + namespace: kube-system + path: "/netdeletion" + caBundle: "{{ .Values.danm_webhook.webhook_ca_bundle }}" + rules: + - operations: ["DELETE"] + apiGroups: ["danm.k8s.io"] + apiVersions: ["v1"] + resources: ["tenantnetworks"] + failurePolicy: Fail +{{ end }} diff --git a/infra-charts/templates/danm-webhook-ds.yaml b/infra-charts/templates/danm-webhook-ds.yaml new file mode 100644 index 0000000..2d898e6 --- /dev/null +++ b/infra-charts/templates/danm-webhook-ds.yaml @@ -0,0 +1,52 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.danm_webhook.required }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: danm-webhook-deployment + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: danm-webhook +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: danm-webhook + template: + metadata: + name: danm-webhook + labels: + {{ .Values.kubernetes_component_label }}: danm-webhook + spec: + serviceAccountName: danm-webhook + nodeSelector: + nodetype: caas_master + containers: + - name: danm-webhook + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/danm-webhook:{{ .Values.danm_webhook.container_version }} + command: [ "/usr/local/bin/webhook", "-tls-cert-bundle=/etc/webhook/certs/danm_webhook.crt", "-tls-private-key-file=/etc/webhook/certs/danm_webhook.key", "-bind-port={{ .Values.danm_webhook.webhook_target_port }}" ] + imagePullPolicy: IfNotPresent + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + # TODO: make it a secret + - name: webhook-certs + hostPath: + path: /etc/kubernetes/ssl/ +{{ end }} diff --git a/infra-charts/templates/danm-webhook-service.yaml b/infra-charts/templates/danm-webhook-service.yaml new file mode 100644 index 0000000..7fda579 --- /dev/null +++ b/infra-charts/templates/danm-webhook-service.yaml @@ -0,0 +1,32 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.danm_webhook.required }} +--- +apiVersion: v1 +kind: Service +metadata: + name: danm-webhook-svc + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: danm-webhook +spec: + ports: + - name: webhook + port: {{ .Values.danm_webhook.webhook_port }} + targetPort: {{ .Values.danm_webhook.webhook_target_port }} + selector: + {{ .Values.kubernetes_component_label }}: danm-webhook +{{ end }} diff --git a/infra-charts/templates/elasticsearch-data-statefulset.yaml b/infra-charts/templates/elasticsearch-data-statefulset.yaml index 5e41391..d9c8326 100644 --- a/infra-charts/templates/elasticsearch-data-statefulset.yaml +++ b/infra-charts/templates/elasticsearch-data-statefulset.yaml @@ -31,13 +31,6 @@ spec: labels: app: elasticsearch-data component: "{{ .Values.elasticsearch.data.name }}" - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] spec: priorityClassName: "system-cluster-critical" affinity: diff --git a/infra-charts/templates/elasticsearch-master-statefulset.yaml b/infra-charts/templates/elasticsearch-master-statefulset.yaml index 99e405b..30ba145 100644 --- a/infra-charts/templates/elasticsearch-master-statefulset.yaml +++ b/infra-charts/templates/elasticsearch-master-statefulset.yaml @@ -31,13 +31,6 @@ spec: labels: app: elasticsearch-master component: "{{ .Values.elasticsearch.master.name }}" - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] spec: priorityClassName: "system-cluster-critical" affinity: diff --git a/infra-charts/templates/fluentd-daemonset.yaml b/infra-charts/templates/fluentd-daemonset.yaml index b709987..b8e41a1 100644 --- a/infra-charts/templates/fluentd-daemonset.yaml +++ b/infra-charts/templates/fluentd-daemonset.yaml @@ -31,12 +31,6 @@ spec: metadata: annotations: config/timestamp: "{{ .Values.fluentd.config_timestamp }}" - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] labels: {{ .Values.kubernetes_component_label }}: fluentd-logging version: v1 diff --git a/infra-charts/templates/metrics-server-deployment.yaml b/infra-charts/templates/metrics-server-deployment.yaml index 91f8197..3f92a6a 100755 --- a/infra-charts/templates/metrics-server-deployment.yaml +++ b/infra-charts/templates/metrics-server-deployment.yaml @@ -28,13 +28,6 @@ spec: {{ .Values.kubernetes_component_label }}: metrics-server template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] name: metrics-server labels: {{ .Values.kubernetes_component_label }}: metrics-server diff --git a/infra-charts/templates/prometheus-dep.yaml b/infra-charts/templates/prometheus-dep.yaml index cedae67..6970921 100644 --- a/infra-charts/templates/prometheus-dep.yaml +++ b/infra-charts/templates/prometheus-dep.yaml @@ -31,12 +31,6 @@ spec: {{ .Values.kubernetes_component_label }}: prometheus annotations: prometheus.io/scrape: 'false' - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] spec: priorityClassName: "system-cluster-critical" nodeSelector: diff --git a/infra-charts/templates/sriovdp-ds.yaml b/infra-charts/templates/sriovdp-ds.yaml index efe37a0..9cb9577 100644 --- a/infra-charts/templates/sriovdp-ds.yaml +++ b/infra-charts/templates/sriovdp-ds.yaml @@ -28,13 +28,6 @@ spec: name: sriovdp template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] labels: name: sriovdp spec: diff --git a/infra-charts/templates/svcwatcher.yaml b/infra-charts/templates/svcwatcher.yaml index 95b18ec..dbb5d22 100644 --- a/infra-charts/templates/svcwatcher.yaml +++ b/infra-charts/templates/svcwatcher.yaml @@ -26,13 +26,6 @@ spec: {{ .Values.kubernetes_component_label }}: svcwatcher template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] labels: {{ .Values.kubernetes_component_label }}: svcwatcher spec: diff --git a/infra-charts/templates/webhook-conf-updater-job.yaml b/infra-charts/templates/webhook-conf-updater-job.yaml index 36a62a6..08ad83a 100644 --- a/infra-charts/templates/webhook-conf-updater-job.yaml +++ b/infra-charts/templates/webhook-conf-updater-job.yaml @@ -11,13 +11,6 @@ metadata: spec: template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] spec: containers: - name: webhook-cfg-updater diff --git a/infra-charts/templates/webhook-deployment.yaml b/infra-charts/templates/webhook-deployment.yaml index 9a3f0f3..eabd01d 100644 --- a/infra-charts/templates/webhook-deployment.yaml +++ b/infra-charts/templates/webhook-deployment.yaml @@ -28,13 +28,6 @@ spec: {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator template: metadata: - annotations: - danm.k8s.io/interfaces: | - [ - { - "network":"flannel" - } - ] name: cpu-dev-pod-mutator labels: {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator diff --git a/infra-charts/values.yaml.j2 b/infra-charts/values.yaml.j2 index 045e107..6698ccd 100644 --- a/infra-charts/values.yaml.j2 +++ b/infra-charts/values.yaml.j2 @@ -59,13 +59,20 @@ cpupooler: cpu_poolconfigs: {{ ansible_local['cpu_pooler_config'] | default([]) | to_nice_yaml | indent(4) }} +danm_webhook: + required: true + container_version: {{ container_image_names | select('search', '/danm-webhook') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + webhook_ca_bundle: {{ webhooks_ca }} + webhook_port: {{ caas.danm_webhook_port }} + webhook_target_port: {{ caas.danm_webhook_target_port }} + elasticsearch: {% set clustermasternodes = [] -%} {%- for masternumber in range(groups['caas_master']|length|int) -%} {%- if clustermasternodes.append('elasticsearch-master-' + (masternumber|string)) -%}{%- endif -%} {%- endfor %} clustermasternodes: '{{ clustermasternodes|join(", ") }}' -{% if caas.infra_log_store is defined and caas.infra_log_store != 'remote_syslog' %} +{% if caas.infra_log_store is defined and caas.infra_log_store != 'remote_syslog' %} required: true {% endif %} url: {{ caas.elasticsearch_url }} -- 2.16.6