From 6eb7d2a455f6e41d02697ef08f2744a972aa69b2 Mon Sep 17 00:00:00 2001 From: "Szekeres, Balazs (Nokia - HU/Budapest)" Date: Thu, 2 May 2019 13:38:38 +0200 Subject: [PATCH] Added seed code for caas-install. Added seed code for caas-install. Change-Id: Ic654b4f0704d373948d7a5c9e08facc94dbec7bf Signed-off-by: Szekeres, Balazs (Nokia - HU/Budapest) --- LICENSE | 179 ++++++++++++ SPECS/infra-charts.spec | 68 +++++ SPECS/instantiate.spec | 110 +++++++ SPECS/utils.spec | 91 ++++++ ansible/filter_plugins/config_helpers.py | 33 +++ ansible/library/kubectl.py | 170 +++++++++++ ansible/playbooks/app_install.yaml | 23 ++ ansible/playbooks/caas_cleanup.yaml | 21 ++ ansible/playbooks/cloud_admin_user.yaml | 23 ++ ansible/playbooks/common.yaml | 22 ++ ansible/playbooks/docker.yaml | 22 ++ ansible/playbooks/image_push.yaml | 30 ++ ansible/playbooks/install_caas_infra.yaml | 29 ++ ansible/playbooks/openrc_hack.yaml | 29 ++ ansible/playbooks/pre_config_all.yaml | 22 ++ ansible/roles/app_install/tasks/main.yml | 58 ++++ ansible/roles/caas_cleanup/tasks/main.yml | 52 ++++ ansible/roles/cloud_admin_user/tasks/main.yml | 22 ++ .../common_tasks/tasks/what_is_the_filesystem.yml | 41 +++ ansible/roles/docker/defaults/main.yaml | 16 + ansible/roles/docker/meta/main.yml | 73 +++++ ansible/roles/docker/tasks/configure_docker.yml | 51 ++++ ansible/roles/docker/tasks/main.yml | 46 +++ ansible/roles/docker/templates/docker-storage | 16 + ansible/roles/docker/templates/docker.service | 63 ++++ ansible/roles/docker/templates/docker.socket | 27 ++ ansible/roles/docker_image_load/defaults/main.yaml | 24 ++ ansible/roles/docker_image_load/tasks/main.yaml | 81 ++++++ ansible/roles/docker_image_push/defaults/main.yaml | 22 ++ ansible/roles/docker_image_push/tasks/main.yml | 55 ++++ .../roles/install_caas_infra/defaults/main.yaml | 24 ++ ansible/roles/install_caas_infra/meta/main.yaml | 38 +++ ansible/roles/install_caas_infra/tasks/main.yaml | 52 ++++ ansible/roles/manifests/tasks/main.yaml | 32 ++ ansible/roles/nodeconf/tasks/load_cacert.yml | 36 +++ ansible/roles/nodeconf/tasks/main.yml | 40 +++ ansible/roles/nodeconf/templates/hosts | 22 ++ ansible/roles/nodeconf/templates/resolv.conf | 22 ++ ansible/roles/pre_config_all/tasks/main.yml | 155 ++++++++++ .../pre_config_all/templates/podruntime.slice | 22 ++ .../roles/pre_install_caas_infra/meta/main.yaml | 52 ++++ .../roles/pre_install_caas_infra/tasks/main.yaml | 20 ++ .../tasks/prepare_cpupooler.yaml | 25 ++ .../tasks/prepare_metrics.yaml | 50 ++++ cm_config/caas.yaml | 102 +++++++ infra-charts/Chart.yaml | 22 ++ .../templates/cm-adapter-serving-certs.yaml | 26 ++ .../templates/cpu-deviceplugin-daemonset.yaml | 66 +++++ infra-charts/templates/cpu-pooler-configmap.yaml | 28 ++ infra-charts/templates/cpu-setter-daemonset.yaml | 65 +++++ .../custom-metrics-apiserver-config-map.yaml | 92 ++++++ .../custom-metrics-apiserver-deployment.yaml | 94 ++++++ .../custom-metrics-apiserver-service.yaml | 29 ++ .../templates/custom-metrics-apiservice.yaml | 31 ++ .../templates/elasticsearch-configmap.yaml | 149 ++++++++++ .../templates/elasticsearch-data-statefulset.yaml | 122 ++++++++ .../elasticsearch-master-statefulset.yaml | 108 +++++++ infra-charts/templates/elasticsearch-service.yaml | 49 ++++ infra-charts/templates/metrics-apiservice.yaml | 31 ++ .../templates/metrics-server-deployment.yaml | 61 ++++ infra-charts/templates/metrics-server-service.yaml | 32 ++ infra-charts/templates/netwatcher.yaml | 49 ++++ infra-charts/templates/prometheus-cfg.yaml | 321 +++++++++++++++++++++ infra-charts/templates/prometheus-dep.yaml | 94 ++++++ .../templates/prometheus-serving-certs.yaml | 26 ++ infra-charts/templates/prometheus-svc.yaml | 32 ++ infra-charts/templates/sriovdp.yml | 71 +++++ infra-charts/templates/svcwatcher.yaml | 54 ++++ infra-charts/templates/webhook-conf.yaml | 40 +++ infra-charts/templates/webhook-deployment.yaml | 63 ++++ infra-charts/templates/webhook-service.yaml | 31 ++ infra-charts/values.yaml.j2 | 82 ++++++ utils/common/aliases.sh | 19 ++ utils/deploy/merge_image.sh | 96 ++++++ utils/log/kubelet-plugins | 32 ++ 75 files changed, 4226 insertions(+) create mode 100644 LICENSE create mode 100644 SPECS/infra-charts.spec create mode 100644 SPECS/instantiate.spec create mode 100644 SPECS/utils.spec create mode 100755 ansible/filter_plugins/config_helpers.py create mode 100644 ansible/library/kubectl.py create mode 100644 ansible/playbooks/app_install.yaml create mode 100644 ansible/playbooks/caas_cleanup.yaml create mode 100644 ansible/playbooks/cloud_admin_user.yaml create mode 100644 ansible/playbooks/common.yaml create mode 100644 ansible/playbooks/docker.yaml create mode 100644 ansible/playbooks/image_push.yaml create mode 100644 ansible/playbooks/install_caas_infra.yaml create mode 100644 ansible/playbooks/openrc_hack.yaml create mode 100644 ansible/playbooks/pre_config_all.yaml create mode 100644 ansible/roles/app_install/tasks/main.yml create mode 100644 ansible/roles/caas_cleanup/tasks/main.yml create mode 100644 ansible/roles/cloud_admin_user/tasks/main.yml create mode 100644 ansible/roles/common_tasks/tasks/what_is_the_filesystem.yml create mode 100644 ansible/roles/docker/defaults/main.yaml create mode 100644 ansible/roles/docker/meta/main.yml create mode 100644 ansible/roles/docker/tasks/configure_docker.yml create mode 100755 ansible/roles/docker/tasks/main.yml create mode 100644 ansible/roles/docker/templates/docker-storage create mode 100644 ansible/roles/docker/templates/docker.service create mode 100644 ansible/roles/docker/templates/docker.socket create mode 100644 ansible/roles/docker_image_load/defaults/main.yaml create mode 100644 ansible/roles/docker_image_load/tasks/main.yaml create mode 100644 ansible/roles/docker_image_push/defaults/main.yaml create mode 100644 ansible/roles/docker_image_push/tasks/main.yml create mode 100644 ansible/roles/install_caas_infra/defaults/main.yaml create mode 100644 ansible/roles/install_caas_infra/meta/main.yaml create mode 100644 ansible/roles/install_caas_infra/tasks/main.yaml create mode 100644 ansible/roles/manifests/tasks/main.yaml create mode 100644 ansible/roles/nodeconf/tasks/load_cacert.yml create mode 100644 ansible/roles/nodeconf/tasks/main.yml create mode 100644 ansible/roles/nodeconf/templates/hosts create mode 100644 ansible/roles/nodeconf/templates/resolv.conf create mode 100644 ansible/roles/pre_config_all/tasks/main.yml create mode 100644 ansible/roles/pre_config_all/templates/podruntime.slice create mode 100644 ansible/roles/pre_install_caas_infra/meta/main.yaml create mode 100644 ansible/roles/pre_install_caas_infra/tasks/main.yaml create mode 100644 ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml create mode 100644 ansible/roles/pre_install_caas_infra/tasks/prepare_metrics.yaml create mode 100644 cm_config/caas.yaml create mode 100644 infra-charts/Chart.yaml create mode 100644 infra-charts/templates/cm-adapter-serving-certs.yaml create mode 100644 infra-charts/templates/cpu-deviceplugin-daemonset.yaml create mode 100644 infra-charts/templates/cpu-pooler-configmap.yaml create mode 100644 infra-charts/templates/cpu-setter-daemonset.yaml create mode 100644 infra-charts/templates/custom-metrics-apiserver-config-map.yaml create mode 100755 infra-charts/templates/custom-metrics-apiserver-deployment.yaml create mode 100755 infra-charts/templates/custom-metrics-apiserver-service.yaml create mode 100755 infra-charts/templates/custom-metrics-apiservice.yaml create mode 100644 infra-charts/templates/elasticsearch-configmap.yaml create mode 100644 infra-charts/templates/elasticsearch-data-statefulset.yaml create mode 100644 infra-charts/templates/elasticsearch-master-statefulset.yaml create mode 100644 infra-charts/templates/elasticsearch-service.yaml create mode 100755 infra-charts/templates/metrics-apiservice.yaml create mode 100755 infra-charts/templates/metrics-server-deployment.yaml create mode 100755 infra-charts/templates/metrics-server-service.yaml create mode 100644 infra-charts/templates/netwatcher.yaml create mode 100644 infra-charts/templates/prometheus-cfg.yaml create mode 100644 infra-charts/templates/prometheus-dep.yaml create mode 100644 infra-charts/templates/prometheus-serving-certs.yaml create mode 100644 infra-charts/templates/prometheus-svc.yaml create mode 100644 infra-charts/templates/sriovdp.yml create mode 100644 infra-charts/templates/svcwatcher.yaml create mode 100644 infra-charts/templates/webhook-conf.yaml create mode 100644 infra-charts/templates/webhook-deployment.yaml create mode 100644 infra-charts/templates/webhook-service.yaml create mode 100644 infra-charts/values.yaml.j2 create mode 100644 utils/common/aliases.sh create mode 100644 utils/deploy/merge_image.sh create mode 100644 utils/log/kubelet-plugins diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4959a5e --- /dev/null +++ b/LICENSE @@ -0,0 +1,179 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec new file mode 100644 index 0000000..d5986cb --- /dev/null +++ b/SPECS/infra-charts.spec @@ -0,0 +1,68 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define COMPONENT infra-charts +%define RPM_NAME caas-%{COMPONENT} +%define RPM_MAJOR_VERSION 1.0.0 +%define RPM_MINOR_VERSION 1 +%define CHART_DIR /var/lib/caas/infra-charts/ + +Name: %{RPM_NAME} +Version: %{RPM_MAJOR_VERSION} +Release: %{RPM_MINOR_VERSION}%{?dist} +Summary: Containers as a Service helm charts +License: %{_platform_license} +BuildArch: x86_64 +Vendor: %{_platform_vendor} +Source0: %{name}-%{version}.tar.gz + +%description +This rpm contains the necessary helm charts to deploy the caas subsystem. + +%prep +%autosetup + +%build + +%install +mkdir -p %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/install_caas_infra.yaml %{buildroot}/%{_playbooks_path}/ + +mkdir -p %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/install_caas_infra %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/pre_install_caas_infra %{buildroot}/%{_roles_path}/ + +mkdir -p %{buildroot}/%{CHART_DIR}/ +rsync -av infra-charts/* %{buildroot}/%{CHART_DIR}/ + +%files +%{_playbooks_path}/* +%{_roles_path}/* +%{CHART_DIR}/* + + +%preun + +%post +mkdir -p %{_postconfig_path}/ +ln -sf %{_playbooks_path}/install_caas_infra.yaml %{_postconfig_path}/ + + +%postun +if [ $1 -eq 0 ]; then + rm -f %{_postconfig_path}/install_caas_infra.yaml +fi + +%clean +rm -rf ${buildroot} diff --git a/SPECS/instantiate.spec b/SPECS/instantiate.spec new file mode 100644 index 0000000..2465b0f --- /dev/null +++ b/SPECS/instantiate.spec @@ -0,0 +1,110 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define COMPONENT instantiate +%define RPM_NAME caas-%{COMPONENT} +%define RPM_MAJOR_VERSION 1.0.0 +%define RPM_MINOR_VERSION 1 + +Name: %{RPM_NAME} +Version: %{RPM_MAJOR_VERSION} +Release: %{RPM_MINOR_VERSION}%{?dist} +Summary: Containers as a Service instantiate playbooks +License: %{_platform_license} +BuildArch: x86_64 +Vendor: %{_platform_vendor} +Source0: %{name}-%{version}.tar.gz + +%description +This rpm contains the necessary playbooks to instantiate the caas subsystem. + +%prep +%autosetup + +%build + +%install +mkdir -p %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/app_install.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/caas_cleanup.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/cloud_admin_user.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/common.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/docker.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/image_push.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/openrc_hack.yaml %{buildroot}/%{_playbooks_path}/ +rsync -av ansible/playbooks/pre_config_all.yaml %{buildroot}/%{_playbooks_path}/ + +mkdir -p %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/app_install %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/caas_cleanup %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/cloud_admin_user %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/common_tasks %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/docker %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/docker_image_load %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/docker_image_push %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/manifests %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/nodeconf %{buildroot}/%{_roles_path}/ +rsync -av ansible/roles/pre_config_all %{buildroot}/%{_roles_path}/ + +mkdir -p %{buildroot}%_platform_etc_path/playbooks/bootstrapping/ + +mkdir -p %{buildroot}%/etc/lcm/playbooks/installation/provisioning/ + +mkdir -p %{buildroot}/etc/ansible/roles/plugins/filter/ +rsync -av ansible/filter_plugins/* %{buildroot}/etc/ansible/roles/plugins/filter/ + +mkdir -p %{buildroot}/etc/ansible/roles/plugins/library/ +rsync -av ansible/library/* %{buildroot}/etc/ansible/roles/plugins/library/ + +mkdir -p %{buildroot}/etc/cmframework/config +rsync -av cm_config/caas.yaml %{buildroot}/etc/cmframework/config/caas.yaml + + +%files +%{_playbooks_path}/* +%{_roles_path}/* +/etc/ansible/roles/plugins/filter/* +/etc/ansible/roles/plugins/library/* +/etc/cmframework/config/* + + +%preun + +%post +mkdir -p %{_postconfig_path}/ +ln -sf %{_playbooks_path}/app_install.yaml %{_postconfig_path}/ +ln -sf %{_playbooks_path}/cloud_admin_user.yaml %{_postconfig_path}/ +ln -sf %{_playbooks_path}/common.yaml %{_postconfig_path}/ +ln -sf %{_playbooks_path}/docker.yaml %{_postconfig_path}/ +ln -sf %{_playbooks_path}/image_push.yaml %{_postconfig_path}/ +ln -sf %{_playbooks_path}/openrc_hack.yaml %{_postconfig_path}/ +ln -sf %{_playbooks_path}/pre_config_all.yaml %{_postconfig_path}/ + +mkdir -p %{_finalize_path}/ +ln -sf %{_playbooks_path}/caas_cleanup.yaml     %{_finalize_path}/ + +%postun +if [ $1 -eq 0 ]; then + rm -f %{_postconfig_path}/app_install.yaml + rm -f %{_postconfig_path}/cloud_admin_user.yaml + rm -f %{_postconfig_path}/common.yaml + rm -f %{_postconfig_path}/docker.yaml + rm -f %{_postconfig_path}/image_push.yaml + rm -f %{_postconfig_path}/openrc_hack.yaml + rm -f %{_postconfig_path}/pre_config_all.yaml + rm -f %{_finalize_path}/caas_cleanup.yaml +fi + +%clean +rm -rf ${buildroot} diff --git a/SPECS/utils.spec b/SPECS/utils.spec new file mode 100644 index 0000000..d504dfe --- /dev/null +++ b/SPECS/utils.spec @@ -0,0 +1,91 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define COMPONENT utils +%define RPM_NAME caas-%{COMPONENT} +%define RPM_MAJOR_VERSION 1.0.0 +%define RPM_MINOR_VERSION 1 +%define LIBEXEC_DIR /usr/libexec/nokia/ +%define KUBELET_PLUGINS_LOGDIR /var/log/kubelet-plugins/ + +Name: %{RPM_NAME} +Version: %{RPM_MAJOR_VERSION} +Release: %{RPM_MINOR_VERSION}%{?dist} +Summary: Containers as a Service supplementary utils +License: %{_platform_license} +BuildArch: x86_64 +Vendor: %{_platform_vendor} +Source0: %{name}-%{version}.tar.gz + +Requires: initscripts + +%description +This rpm contains the supplementary utils for caas subsystem. + +%prep + +%autosetup + +%build + +%install +mkdir -p %{buildroot}/%{LIBEXEC_DIR}/ +# --------------------------- LOG +mkdir -p %{buildroot}/etc/logrotate.d/ +install -m 0640 utils/log/kubelet-plugins %{buildroot}/etc/logrotate.d/ +sed -i -e 's|{{ kubelet_plugings_log_dir }}|%{KUBELET_PLUGINS_LOGDIR}|g' %{buildroot}/etc/logrotate.d/kubelet-plugins +# --------------------------- DEPLOY +install -m 0700 utils/deploy/merge_image.sh %{buildroot}/%{LIBEXEC_DIR}/ +mkdir -p %{buildroot}/etc/systemd/system/ +# --------------------------- COMMON +mkdir -p %{buildroot}/etc/profile.d/ +install -m 0644 utils/common/aliases.sh %{buildroot}/etc/profile.d/ + +%files +%{LIBEXEC_DIR}/merge_image.sh +/etc/profile.d/aliases.sh +/etc/logrotate.d/kubelet-plugins +%exclude %{LIBEXEC_DIR}/*pyc +%exclude %{LIBEXEC_DIR}/*pyo + +%preun + +%post +# --------------------------- LOG +mkdir -p %{KUBELET_PLUGINS_LOGDIR}/ +grep "#CaaS CUSTOM BEGIN" /etc/logrotate.d/syslog > /dev/null; +if [ $? -eq 0 ]; then + sed -i -e '/#CaaS CUSTOM BEGIN/,/#CaaS CUSTOM END/d' /etc/logrotate.d/syslog + fi +sed -i.bak -e '/.*missingok/i #CaaS CUSTOM BEGIN\n hourly\n size 50\n#CaaS CUSTOM END' /etc/logrotate.d/syslog +# --------------------------- DEPLOY +find /usr/lib/debug/usr/ -xtype l -exec rm -f {} \; +# --------------------------- COMMON + +%postun +# If not upgrade, revert all CaaS related cusotmization +if [ $1 == 0 ]; then + +# --------------------------- LOG + sed -i -e '/#CaaS CUSTOM BEGIN/,/#CaaS CUSTOM END/d' /etc/logrotate.d/syslog +/usr/bin/systemctl daemon-reload +# --------------------------- DEPLOY +# --------------------------- COMMON + rm -rf /etc/profile.d/aliases.sh + +fi + + +%clean +rm -rf ${buildroot} diff --git a/ansible/filter_plugins/config_helpers.py b/ansible/filter_plugins/config_helpers.py new file mode 100755 index 0000000..8e0ab7a --- /dev/null +++ b/ansible/filter_plugins/config_helpers.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.errors import AnsibleError + + +class FilterModule(object): + def filters(self): + return { + 'get_kube_options': get_kube_options, + } + + +def get_kube_options(options): + if not isinstance(options, dict): + raise AnsibleError("Invalid type {}. Options must be dictionary!".format(type(options))) + + option_template = "{}={}" + formated_options = [option_template.format(option, str(value)) + for option, value in options.iteritems()] + return ",".join(formated_options) diff --git a/ansible/library/kubectl.py b/ansible/library/kubectl.py new file mode 100644 index 0000000..f141b33 --- /dev/null +++ b/ansible/library/kubectl.py @@ -0,0 +1,170 @@ +#!/usr/bin/python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import subprocess + +from ansible.module_utils.basic import AnsibleModule + +DOCUMENTATION = ''' +--- +module: kubectl +version_added: "2.4" +short_description: Simply module to manage Kubernetes objects via manifest files +description: +- "Simply module to manage Kubernetes objects via manifest files" +options: + manifest: + description: + - Path of manifest file describing Kubernetes object(s) + required: true + type: str + kubeconfig: + description: + - Path of kubeconfig file used to connect apiserver + required: false + type: str + default: ~/.kube/config + timeout: + description: + - Max reauest timeout in seconds + required: false + type: int + default: 5 + state: + required: false + default: present + choices: ['present', 'absent'] +author: + - krisztian.lengyel@nokia.com +''' + +EXAMPLES = ''' +# Create a pod in Kubernetes +- name: Create pod + kubectl: + manifest: /home/kube_manifests/some-useful-pod.yaml + state: present + +# Delete some service +- name: Delete Kubernetes service + kubectl: + manifest: /home/kube_manifests/some-useless-svc.yaml + kubeconfig: /home/admin-kubeconfig.yaml + state: present + +# Create many object from a single file +- name: Create stuffs + kubectl: + manifest: /home/kube_manifests/so-many-stuff.yaml + timeout: 30 + state: present +''' + +RETURN = ''' +message: + description: The output (stdout & stderr) of kubectl command +''' + +STATE_MAPPING = { + "present": "create", + "absent": "delete" +} + +ALREADY_EXIST_PATTERN = r'Error from server \(AlreadyExists\): error when creating .+ already exists' +SVC_IP_ALREADY_ALLOCATED_PATTERN = r'The Service .+ is invalid: .+ provided IP is already allocated' +NOT_FOUND_PATTERN = r'Error from server \(NotFound\): .+ not found' + +KUBECONFIG_PATH = ".kube/config" + + +class KubectlExecutionError(Exception): + pass + + +def main(): + module = _build_initialized_module() + + _add_defaults_to_params(module.params) + + try: + result = _handle_module_result(*_execute_kubectl(module.params)) + module.exit_json(**result) + except KubectlExecutionError as ex: + module.fail_json(msg=str(ex)) + + +def _build_initialized_module(): + module_args = dict( + manifest=dict(required=True, type='str'), + kubeconfig=dict(required=False, type='str'), + timeout=dict(required=False, type='int', default=5), + state=dict(required=False, choices=['present', 'absent'], type='str', default="present") + ) + + return AnsibleModule( + argument_spec=module_args, + supports_check_mode=False + ) + + +def _add_defaults_to_params(params): + if not params['kubeconfig']: + params['kubeconfig'] = _get_default_kubeconfig_path() + + +def _get_default_kubeconfig_path(): + return "{}/{}".format( + os.environ.get('HOME', ''), + KUBECONFIG_PATH) + + +def _execute_kubectl(params): + kubectl_command = STATE_MAPPING[params['state']] + kubectl_cmd = ("/usr/bin/kubectl", kubectl_command) + _get_kubectl_flags(params) + ansible_process = subprocess.Popen(kubectl_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return ansible_process.communicate() + + +def _get_kubectl_flags(flags): + return ("--kubeconfig={}".format(flags['kubeconfig']), + "--filename={}".format(flags['manifest']), + "--request-timeout={}s".format(flags['timeout'])) + + +def _handle_module_result(output, error): + changed = _is_changed(error) + if changed and error: + raise KubectlExecutionError("Error: {}".format(error)) + + return { + 'changed': changed, + 'message': output + error + } + + +def _is_changed(message): + if (re.search(ALREADY_EXIST_PATTERN, message) + or re.search(SVC_IP_ALREADY_ALLOCATED_PATTERN, message) + or re.search(NOT_FOUND_PATTERN, message)): + return False + return True + + +if __name__ == '__main__': + main() diff --git a/ansible/playbooks/app_install.yaml b/ansible/playbooks/app_install.yaml new file mode 100644 index 0000000..4f6e0d5 --- /dev/null +++ b/ansible/playbooks/app_install.yaml @@ -0,0 +1,23 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: fluentd.yaml +- hosts: installation_controller:&caas_master + strategy: linear + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - role: app_install + when: ( caas.chart_name is defined ) and ( caas.chart_name != "" ) and ( caas.chart_name is not none ) diff --git a/ansible/playbooks/caas_cleanup.yaml b/ansible/playbooks/caas_cleanup.yaml new file mode 100644 index 0000000..44272f6 --- /dev/null +++ b/ansible/playbooks/caas_cleanup.yaml @@ -0,0 +1,21 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: caas_nodes + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - caas_cleanup diff --git a/ansible/playbooks/cloud_admin_user.yaml b/ansible/playbooks/cloud_admin_user.yaml new file mode 100644 index 0000000..6855f92 --- /dev/null +++ b/ansible/playbooks/cloud_admin_user.yaml @@ -0,0 +1,23 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: ntpdate.yaml +- hosts: caas_nodes + strategy: free + become: true + become_user: "root" + roles: + - role: cloud_admin_user + diff --git a/ansible/playbooks/common.yaml b/ansible/playbooks/common.yaml new file mode 100644 index 0000000..0cb6c1d --- /dev/null +++ b/ansible/playbooks/common.yaml @@ -0,0 +1,22 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: cloud_admin_user.yaml +- hosts: caas_nodes + strategy: free + become: true + become_user: "root" + roles: + - role: nodeconf diff --git a/ansible/playbooks/docker.yaml b/ansible/playbooks/docker.yaml new file mode 100644 index 0000000..26ec89b --- /dev/null +++ b/ansible/playbooks/docker.yaml @@ -0,0 +1,22 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: bare_lvm_backend.yml,common.yaml +- hosts: caas_nodes + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - docker diff --git a/ansible/playbooks/image_push.yaml b/ansible/playbooks/image_push.yaml new file mode 100644 index 0000000..bbbb8bf --- /dev/null +++ b/ansible/playbooks/image_push.yaml @@ -0,0 +1,30 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: registry.yaml +- hosts: installation_controller:&caas_master + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - role: docker_image_push + wait_for_registry: true + images: + - hyperkube + - etcd + - kubedns + - flannel + - swift + - registry diff --git a/ansible/playbooks/install_caas_infra.yaml b/ansible/playbooks/install_caas_infra.yaml new file mode 100644 index 0000000..733c270 --- /dev/null +++ b/ansible/playbooks/install_caas_infra.yaml @@ -0,0 +1,29 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: helm.yaml +- hosts: caas_master + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - pre_install_caas_infra + +- hosts: installation_controller:&caas_master + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - install_caas_infra diff --git a/ansible/playbooks/openrc_hack.yaml b/ansible/playbooks/openrc_hack.yaml new file mode 100644 index 0000000..ab22109 --- /dev/null +++ b/ansible/playbooks/openrc_hack.yaml @@ -0,0 +1,29 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: app_install.yaml +- name: Create mandatory openrc file for finalize plays + hosts: installation_controller:&caas_master + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + tasks: + - name: Create mandatory openrc file for finalize plays + file: + path: "/home/{{ users.admin_user_name }}/openrc" + owner: "{{ users.admin_user_name }}" + group: "{{ users.admin_user_name }}" + state: touch + diff --git a/ansible/playbooks/pre_config_all.yaml b/ansible/playbooks/pre_config_all.yaml new file mode 100644 index 0000000..de21880 --- /dev/null +++ b/ansible/playbooks/pre_config_all.yaml @@ -0,0 +1,22 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cmframework.requires: pre_config_OS.yaml +- hosts: caas_nodes + strategy: free + become: true + become_user: "{{ users.admin_user_name }}" + roles: + - pre_config_all diff --git a/ansible/roles/app_install/tasks/main.yml b/ansible/roles/app_install/tasks/main.yml new file mode 100644 index 0000000..1d0a3bf --- /dev/null +++ b/ansible/roles/app_install/tasks/main.yml @@ -0,0 +1,58 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get nodes ip + shell: 'kubectl get no -o jsonpath="[{range .items[*]}{.metadata.name},{end}]"' + register: node_ips + +- name: helm-list + shell: HELM_HOST={{ caas.tiller_ip }}:{{ caas.tiller_port }} helm list --all | grep -c {{ name|lower }} + register: list_result + ignore_errors: yes + when: caas.chart_name is defined and caas.chart_name != "" + +- name: set helm install parameters + set_fact: + helm_install_set: "" + +- name: set helm flags parameter + set_fact: + helm_flags: "" + +- name: set helm flags parameter + set_fact: + helm_flags: "{{ helm_flags }} --version={{ caas.chart_version }}" + when: (caas.chart_version is defined) and (caas.chart_version is not none) and (caas.chart_version != "") + +- name: create values.yaml if needed + copy: + content: | + {{ caas.helm_parameters | to_nice_yaml }} + dest: /tmp/Values.yaml + owner: "{{ users.admin_user_name }}" + group: "{{ users.admin_user_name }}" + mode: 0644 + force: yes + when: (caas.helm_parameters is defined) and (caas.helm_parameters is not none) and (caas.helm_parameters != "") + +- name: set helm flags parameters if needed + set_fact: + helm_flags: "{{ helm_flags }} --values /tmp/Values.yaml" + when: (caas.helm_parameters is defined) and (caas.helm_parameters is not none) and (caas.helm_parameters != "") + +- name: helm-install + shell: | + ansible-playbook -i /opt/openstack-ansible/inventory /opt/nokia/caas_lcm/deploy/playbook_helm_install.yml --become-user={{ users.admin_user_name }} -e "chart_name={{ caas.chart_name }} helm_flags='{{ helm_flags }}' release_name={{ name }} helm_operation_timeout={{ caas.helm_operation_timeout }} cbam=false helm_install_set='{{ helm_install_set }}'" + when: list_result is defined and list_result.rc == 1 diff --git a/ansible/roles/caas_cleanup/tasks/main.yml b/ansible/roles/caas_cleanup/tasks/main.yml new file mode 100644 index 0000000..983e2a8 --- /dev/null +++ b/ansible/roles/caas_cleanup/tasks/main.yml @@ -0,0 +1,52 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable ntpd service + service: + name: ntpd + enabled: yes + become_user: "root" + +- name: removing docker image tar files + file: + state: absent + path: "{{ caas.infra_containers_directory }}" + become_user: "root" + +- name: remove ca.pem.enc + file: + state: absent + path: "/etc/openssl/ca.pem.enc" + no_log: True + become_user: "root" + +- name: remove ca-key.pem.enc + file: + state: absent + path: "/etc/openssl/ca-key.pem.enc" + no_log: True + become_user: "root" + +- name: adding acl for cloud_admin_user to read /var/log + acl: + entity: "{{ users.admin_user_name }}" + etype: user + path: "/var/log" + permissions: "rx" + recursive: yes + state: present + become_user: "root" + ignore_errors: yes + no_log: True diff --git a/ansible/roles/cloud_admin_user/tasks/main.yml b/ansible/roles/cloud_admin_user/tasks/main.yml new file mode 100644 index 0000000..353c309 --- /dev/null +++ b/ansible/roles/cloud_admin_user/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: copy the /etc/skel to {{ users.admin_user_name }} + copy: + src: /etc/skel/ + dest: "/home/{{ users.admin_user_name }}" + group: "{{ users.admin_user_name }}" + owner: "{{ users.admin_user_name }}" + mode: 0640 diff --git a/ansible/roles/common_tasks/tasks/what_is_the_filesystem.yml b/ansible/roles/common_tasks/tasks/what_is_the_filesystem.yml new file mode 100644 index 0000000..bcd57ac --- /dev/null +++ b/ansible/roles/common_tasks/tasks/what_is_the_filesystem.yml @@ -0,0 +1,41 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: retrieve filesystem + shell: file -Ls {{device}} | grep "filesystem" + register: filesystem_tmp + failed_when: filesystem_tmp.rc != 0 and filesystem_tmp.rc != 1 + become: true + become_user: "root" + +- debug: + msg: "Filesystem not found on device {{device}}!" + when: filesystem_tmp.rc == 1 + +- name: set filesystem variable + set_fact: + filesystem: "{{ filesystem_tmp }}" + +- name: retrieve filesystem_type + shell: file -Ls "{{device}}" | awk -F" filesystem " '{print($1)}' | awk -F" " '{print($NF)}' + register: filesystem_type_tmp + when: filesystem_tmp.rc == 0 + become: true + become_user: "root" + +- name: set filesystem type variable + set_fact: + filesystem_type: "{{ filesystem_type_tmp }}" + when: filesystem_tmp.rc == 0 diff --git a/ansible/roles/docker/defaults/main.yaml b/ansible/roles/docker/defaults/main.yaml new file mode 100644 index 0000000..447e9ad --- /dev/null +++ b/ansible/roles/docker/defaults/main.yaml @@ -0,0 +1,16 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + container_images_fact_file: "/etc/ansible/facts.d/ncir_container_images.fact" diff --git a/ansible/roles/docker/meta/main.yml b/ansible/roles/docker/meta/main.yml new file mode 100644 index 0000000..79bd0dc --- /dev/null +++ b/ansible/roles/docker/meta/main.yml @@ -0,0 +1,73 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - role: creategroup + _name: kube + _gid: "{{ caas.uid.kube }}" + become: true + become_user: "root" + + - role: createuser + _name: kube + _group: kube + _groups: '' + _shell: /sbin/nologin + _home: / + _uid: "{{ caas.uid.kube }}" + become: true + become_user: "root" + + - role: cert + instance: "docker{{ nodeindex }}" + cert_path: /etc/docker + become: true + become_user: "root" + + - role: cert + cert_name: "cert.pem" + key_name: "key.pem" + instance: "docker" + cert_path: /etc/docker + alt_names: + ip: + - "{{ ansible_host }}" + become: true + become_user: "root" + + - role: cert + instance: "client{{ nodeindex }}" + cert_path: /etc/docker + add_users: + - kube + become: true + become_user: "root" + + - role: creategroup + _name: docker + _gid: "{{ caas.uid.docker }}" + become: true + become_user: "root" + + - role: createuser + _name: docker + _group: docker + _groups: '' + _shell: /sbin/nologin + _home: "{{ caas.docker_directory }}" + _uid: "{{ caas.uid.docker }}" + become: true + become_user: "root" + diff --git a/ansible/roles/docker/tasks/configure_docker.yml b/ansible/roles/docker/tasks/configure_docker.yml new file mode 100644 index 0000000..c911ebb --- /dev/null +++ b/ansible/roles/docker/tasks/configure_docker.yml @@ -0,0 +1,51 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: template docker env files + template: + src: "{{ item }}" + dest: "/etc/sysconfig/{{ item }}" + with_items: + - docker-storage + +- name: template docker socket file + template: + src: docker.socket + dest: /etc/systemd/system/docker.socket + +- name: template docker service file + template: + src: docker.service + dest: /etc/systemd/system/docker.service + +- name: copying ca file to trusted sources + copy: + src: /etc/openssl/ca.pem + dest: /etc/pki/ca-trust/source/anchors/ca.pem + mode: 0400 + +- name: reloading trusted certificates + shell: update-ca-trust + +- name: make env vars permanent + lineinfile: + dest: /etc/environment + state: present + line: "{{ item }}" + regexp: "{{ item }}" + with_items: + - "DOCKER_HOST=tcp://{{ networking.infra_internal.ip }}:2375" + - "DOCKER_TLS_VERIFY=1" + - "DOCKER_CERT_PATH=/etc/docker" diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml new file mode 100755 index 0000000..da98b49 --- /dev/null +++ b/ansible/roles/docker/tasks/main.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- import_tasks: configure_docker.yml + become_user: "root" + +- name: daemon reload + command: /usr/bin/systemctl daemon-reload + become_user: "root" + +- name: restart docker + service: + name: docker + state: started + enabled: yes + become_user: "root" + +- name: restart docker socket + service: + name: docker.socket + state: started + become_user: "root" + +- name: make env vars permanent + lineinfile: + dest: /etc/profile + state: present + line: "{{ item }}" + regexp: "{{ item }}" + with_items: + - "export DOCKER_HOST=tcp://{{ networking.infra_internal.ip }}:2375" + - "export DOCKER_TLS_VERIFY='1'" + - "export DOCKER_CERT_PATH=/etc/docker" + become_user: "root" diff --git a/ansible/roles/docker/templates/docker-storage b/ansible/roles/docker/templates/docker-storage new file mode 100644 index 0000000..e50d318 --- /dev/null +++ b/ansible/roles/docker/templates/docker-storage @@ -0,0 +1,16 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +DOCKER_STORAGE_OPTIONS="--storage-driver overlay2 --storage-opt overlay2.size={{ caas.docker_size_quota }}" diff --git a/ansible/roles/docker/templates/docker.service b/ansible/roles/docker/templates/docker.service new file mode 100644 index 0000000..2e20c3e --- /dev/null +++ b/ansible/roles/docker/templates/docker.service @@ -0,0 +1,63 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +After=network.target docker.socket +Requires=docker.socket +Before=kubelet.service +Wants=kubelet.service +BindsTo=containerd.service + +[Service] +Slice=podruntime.slice +MemoryAccounting=true +CPUAccounting=true +BlockIOAccounting=true +TasksAccounting=true +Type=notify +EnvironmentFile=-/etc/sysconfig/docker-storage +EnvironmentFile=-/etc/sysconfig/docker-proxy +ExecStart=/usr/bin/dockerd -H fd:// \ + -H tcp://{{ ansible_host }}:2375 \ + --max-concurrent-downloads 10 \ + --max-concurrent-uploads 10 \ + --data-root {{ caas.docker_directory }} \ + --tlsverify \ + --tlscacert=/etc/docker/ca.pem \ + --tlscert=/etc/docker/cert.pem \ + --tlskey=/etc/docker/key.pem \ + --log-driver=json-file \ + --log-opt max-size=1g \ + --log-opt max-file=4 \ + --ip-forward=false \ + --live-restore=true \ + --bip={{ caas.docker0_cidr | default('172.17.0.1/16', true) }} \ + --exec-opt native.cgroupdriver=cgroupfs \ + $DOCKER_STORAGE_OPTIONS +MountFlags=shared +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=0 +Restart=on-failure +RestartSec=1s +Environment=DOCKER_HOST=tcp://{{ hostname|lower }}:2375 +Environment=DOCKER_TLS_VERIFY=1 +Environment=DOCKER_CERT_PATH=/etc/docker + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/docker/templates/docker.socket b/ansible/roles/docker/templates/docker.socket new file mode 100644 index 0000000..c7ac7a7 --- /dev/null +++ b/ansible/roles/docker/templates/docker.socket @@ -0,0 +1,27 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +[Unit] +Description=Docker Socket for the API +PartOf=docker.service + +[Socket] +ListenStream=/var/run/docker.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker + +[Install] +WantedBy=sockets.target diff --git a/ansible/roles/docker_image_load/defaults/main.yaml b/ansible/roles/docker_image_load/defaults/main.yaml new file mode 100644 index 0000000..2c51cbc --- /dev/null +++ b/ansible/roles/docker_image_load/defaults/main.yaml @@ -0,0 +1,24 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker_environment: + DOCKER_HOST: "tcp://{{ networking.infra_internal.ip }}:2375" + DOCKER_TLS_VERIFY: "1" + DOCKER_CERT_PATH: "/etc/docker" + +image_file_extension: .tar +image_directory: "{{ caas.infra_containers_directory }}" +registry: "{{ caas.registry_url }}:{{ caas.registry_port }}" +registry_repo: "{{ caas.registry_repo }}" diff --git a/ansible/roles/docker_image_load/tasks/main.yaml b/ansible/roles/docker_image_load/tasks/main.yaml new file mode 100644 index 0000000..bb71634 --- /dev/null +++ b/ansible/roles/docker_image_load/tasks/main.yaml @@ -0,0 +1,81 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: these should be changed to use docker image module. However, the name and the tag should be stored and replaced separately in Ansible dictionary to be able to do that +- name: import docker containers + shell: "/usr/bin/docker load -q -i {{ image_directory }}/{{ item }}*{{ image_file_extension }}" + register: loaded_containers + with_items: "{{ images | mandatory }}" + environment: "{{ docker_environment }}" + +- name: parse loaded images' full name + register: parsed_containers_full_names + set_fact: + image_full_name: "{{ item.stdout | regex_replace('Loaded image: (?P[\\w\\.\\-:\\/_]+)', '\\g') }}" + with_items: + "{{ loaded_containers.results }}" + +- name: parse loaded images' base name + set_fact: + # getting the name of image, then strip the repo path and the tag from it + image_base_name: "{{ item.stdout | regex_replace('Loaded image: ([\\w\\.\\:]+/)?(?P[\\w\\.\\-:\\/_]+)', '\\g') | regex_replace('(:[\\w\\-_\\.]+)$', '') | basename }}" + register: parsed_image_base_names + with_items: + "{{ loaded_containers.results }}" + +- name: parse loaded images' tag + set_fact: + # getting the name of image + image_tag: "{{ item.stdout | regex_replace('Loaded image: ([\\w\\.\\:]+/)?(?P[\\w\\.\\-\\/_]+)', '\\g') | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }}" + register: parsed_image_tags + with_items: + "{{ loaded_containers.results }}" + +- name: save the parsed infra image names + set_fact: + loaded_images: "{{ parsed_containers_full_names.results | map(attribute='ansible_facts.image_full_name') | list }}" + loaded_base_images: "{{ parsed_image_base_names.results | map(attribute='ansible_facts.image_base_name') | list }}" + loaded_image_tags: "{{ parsed_image_tags.results | map(attribute='ansible_facts.image_tag') | list }}" + +- name: assemble new image names + set_fact: + image_name: "{{ registry }}/{{ registry_repo }}/{{ item.0 }}:{{ item.1 }}" + register: assembled_new_images + with_together: + - "{{ loaded_base_images }}" + - "{{ loaded_image_tags }}" + +- name: save retagged infra image names + set_fact: + retagged_images: "{{ assembled_new_images.results | map(attribute='ansible_facts.image_name') | list }}" + +- name: retag loaded images + shell: "/usr/bin/docker tag {{ item.0 }} {{ item.1 }}" + with_together: + - "{{ loaded_images }}" + - "{{ retagged_images }}" + environment: "{{ docker_environment }}" + +- name: delete loaded image's tag + shell: "/usr/bin/docker image rm {{ item }}" + with_items: "{{ loaded_images }}" + when: item not in retagged_images + environment: "{{ docker_environment }}" + +# TODO: Save container images as a dictionary +- name: save the parsed image names + set_fact: + container_image_names: "{{ container_image_names | default([]) }} + {{ retagged_images }}" + diff --git a/ansible/roles/docker_image_push/defaults/main.yaml b/ansible/roles/docker_image_push/defaults/main.yaml new file mode 100644 index 0000000..d6fccf1 --- /dev/null +++ b/ansible/roles/docker_image_push/defaults/main.yaml @@ -0,0 +1,22 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker_environment: + DOCKER_HOST: "tcp://{{ networking.infra_internal.ip }}:2375" + DOCKER_TLS_VERIFY: "1" + DOCKER_CERT_PATH: "/etc/docker" + +registry: "{{ caas.registry_url }}:{{ caas.registry_port }}" +registry_repo: "{{ caas.registry_repo }}" diff --git a/ansible/roles/docker_image_push/tasks/main.yml b/ansible/roles/docker_image_push/tasks/main.yml new file mode 100644 index 0000000..5994f84 --- /dev/null +++ b/ansible/roles/docker_image_push/tasks/main.yml @@ -0,0 +1,55 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- block: + - name: wait for registry endpoints + shell: "kubectl get endpoints registry --namespace=kube-system | grep -o ':{{ caas.registry_port }}' | wc -l" + register: result + until: result.stdout|int == groups.caas_master|length|int + retries: "{{ caas.container_wait_timeout }}" + delay: 1 + + - name: get registry containers ip and port + shell: "kubectl get endpoints registry --namespace=kube-system -o 'jsonpath={.subsets[*].addresses[*].ip}'" + register: registry_ep_ips + + - name: wait for registry containers to start + wait_for: + host: "{{ item }}" + port: "{{ caas.registry_port }}" + state: started + timeout: "{{ caas.container_wait_timeout }}" + with_items: '{{ registry_ep_ips.stdout.split(" ") }}' + when: wait_for_registry | default(False) + +- name: get full image name + shell: "docker images --filter='reference={{ registry }}/{{ registry_repo }}/{{ item }}*' --format='{{ '{{.Repository}}:{{.Tag}}' }}'" + environment: "{{ docker_environment }}" + register: full_image_name + with_items: "{{ images }}" + +- name: save full image names + set_fact: + images_to_push: "{{ full_image_name.results | map(attribute='stdout') | list }}" + +# TODO: these should be changed to use docker image module. However, the name and the tag should be stored and replaced separately in Ansible dictionary to be able to do that +- name: push images + shell: 'docker push "{{ item }}"' + environment: "{{ docker_environment }}" + with_items: "{{ images_to_push }}" + register: push_result + until: push_result.rc == 0 + retries: 5 + delay: 1 diff --git a/ansible/roles/install_caas_infra/defaults/main.yaml b/ansible/roles/install_caas_infra/defaults/main.yaml new file mode 100644 index 0000000..c1b160d --- /dev/null +++ b/ansible/roles/install_caas_infra/defaults/main.yaml @@ -0,0 +1,24 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +infra_chart_images: + - cpupooler + - custom_metrics + - elasticsearch + - metrics_server + - prometheus + - sriovdp + - svcwatcher + - netwatcher diff --git a/ansible/roles/install_caas_infra/meta/main.yaml b/ansible/roles/install_caas_infra/meta/main.yaml new file mode 100644 index 0000000..b48c79c --- /dev/null +++ b/ansible/roles/install_caas_infra/meta/main.yaml @@ -0,0 +1,38 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - role: docker_image_load + images: "{{ infra_chart_images }}" + + - role: docker_image_push + images: "{{ infra_chart_images }}" + when: nodename | search("caas_master1") + + - role: creategroup + _name: elasticsearch + _gid: "{{ caas.uid.elasticsearch }}" + become: true + become_user: "root" + + - role: createuser + _name: elasticsearch + _group: elasticsearch + _groups: '' + _shell: /sbin/nologin + _home: /home/elasticsearch + _uid: "{{ caas.uid.elasticsearch }}" + become: true + become_user: "root" diff --git a/ansible/roles/install_caas_infra/tasks/main.yaml b/ansible/roles/install_caas_infra/tasks/main.yaml new file mode 100644 index 0000000..0e2c856 --- /dev/null +++ b/ansible/roles/install_caas_infra/tasks/main.yaml @@ -0,0 +1,52 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: template values.yaml in helm dir + template: + src: "{{ caas.caas_chart_path }}/values.yaml.j2" + dest: "{{ caas.caas_chart_path }}/values.yaml" + +- set_fact: + infra_chart_name: "caas-infra" + +- set_fact: + chart_file: "{{ lookup('file', '{{ caas.caas_chart_path}}/Chart.yaml') | from_yaml }}" + +- name: create chart tgz + shell: "tar -C {{ caas.caas_chart_path | dirname }} -cvzf {{ infra_chart_name }}-{{ chart_file.version }}.tgz {{ caas.caas_chart_path | basename }}" + +- name: upload individual charts to chart repo + shell: "curl -sS -XPOST -H 'Content-Type: application/gzip' --data-binary @{{ infra_chart_name }}-{{ chart_file.version }}.tgz {{ caas.chart_repo }}/charts/{{ infra_chart_name }}-{{ chart_file.version }}.tgz" + +- name: update helm repo + shell: HELM_HOST={{ caas.tiller_ip }}:{{ caas.tiller_port }} helm repo update + +- name: helm-list + shell: HELM_HOST={{ caas.tiller_ip }}:{{ caas.tiller_port }} helm get {{ infra_chart_name }} + register: crf_list_result + failed_when: "( crf_list_result.rc != 0 ) and ( not (crf_list_result.stderr | search('Error: release')) )" + +- block: + - name: Install infra_chart with helm + shell: HELM_HOST={{ caas.tiller_ip }}:{{ caas.tiller_port }} helm install --wait --timeout {{ caas.helm_operation_timeout }} --name {{ infra_chart_name }} default/{{ infra_chart_name }} + when: "crf_list_result is defined and ( ( crf_list_result.stderr | search('Error: release:') ) or ( crf_list_result.stdout == '' ) )" + + rescue: + - name: Delete the crf_infra_chart + shell: HELM_HOST={{ caas.tiller_ip }}:{{ caas.tiller_port }} helm delete --purge {{ infra_chart_name }} + + - name: fail the current playbook run, because helm install failed + fail: + msg: "ERROR: helm install {{ infra_chart_name }} failed" diff --git a/ansible/roles/manifests/tasks/main.yaml b/ansible/roles/manifests/tasks/main.yaml new file mode 100644 index 0000000..a07ec3e --- /dev/null +++ b/ansible/roles/manifests/tasks/main.yaml @@ -0,0 +1,32 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: create the manifest directory + file: + path: "{{ caas.manifests_directory }}" + state: directory + recurse: yes + become: true + become_user: root + +- name: allowing users.admin_user_name to access docker directory infra + acl: + name: "{{ caas.manifests_directory }}" + entity: "{{ users.admin_user_name }}" + etype: user + permissions: rwx + state: present + become: true + become_user: root diff --git a/ansible/roles/nodeconf/tasks/load_cacert.yml b/ansible/roles/nodeconf/tasks/load_cacert.yml new file mode 100644 index 0000000..8c4c93a --- /dev/null +++ b/ansible/roles/nodeconf/tasks/load_cacert.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + - name: decrypt ca and ca-key + command: "{{ item }}" + with_items: + - "/usr/bin/openssl enc -d -aes-256-cbc -a -in ca-key.pem.enc -pass pass:{{ caas.salt }} -out ca-key.pem" + - "/usr/bin/openssl enc -d -aes-256-cbc -a -in ca.pem.enc -pass pass:{{ caas.salt }} -out ca.pem" + args: + chdir: "/etc/openssl" + no_log: True + + - name: chmod 555 ca.pem + file: + path: /etc/openssl/ca.pem + mode: 0555 # Certs are public. And read by other parties indeed: swift and kubernetes + no_log: True + + - name: chmod 000 ca-key.pem + file: + path: /etc/openssl/ca-key.pem + mode: 0000 # Used only when signing cert by root. + no_log: True + diff --git a/ansible/roles/nodeconf/tasks/main.yml b/ansible/roles/nodeconf/tasks/main.yml new file mode 100644 index 0000000..3d76d49 --- /dev/null +++ b/ansible/roles/nodeconf/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: template resolv.conf + template: + src: resolv.conf + dest: /etc/resolv.conf + +- name: insert host shortcuts + lineinfile: + path: /etc/hosts + line: '\1 {{ caas.apiserver_in_hosts }}\2' + regexp: '(.*){{ item }}(.*)' + backrefs: yes + register: result + with_items: + "{{ groups.caas_master }}" + + +- name: template host shortcuts if needed + template: + src: hosts + dest: /etc/hosts + mode: 0644 + when: not result.changed + + +- import_tasks: load_cacert.yml diff --git a/ansible/roles/nodeconf/templates/hosts b/ansible/roles/nodeconf/templates/hosts new file mode 100644 index 0000000..847d488 --- /dev/null +++ b/ansible/roles/nodeconf/templates/hosts @@ -0,0 +1,22 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 + +{{ ansible_host }} {{ hostname|lower }} + +{% for host in groups['caas_master']%} +{{ hostvars[host]['networking']['infra_internal']['ip'] }} controller-{{loop.index}} {{ caas.apiserver_in_hosts }} +{% endfor %} diff --git a/ansible/roles/nodeconf/templates/resolv.conf b/ansible/roles/nodeconf/templates/resolv.conf new file mode 100644 index 0000000..e6690c9 --- /dev/null +++ b/ansible/roles/nodeconf/templates/resolv.conf @@ -0,0 +1,22 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +# Generated by Ansible +{% for host in groups['caas_master']%} +nameserver {{ hostvars[host]['networking']['infra_internal']['ip'] }} +{% endfor %} +{% if groups['caas_master']|length|int > 1 %} +options rotate +{% endif%} diff --git a/ansible/roles/pre_config_all/tasks/main.yml b/ansible/roles/pre_config_all/tasks/main.yml new file mode 100644 index 0000000..5eaf89d --- /dev/null +++ b/ansible/roles/pre_config_all/tasks/main.yml @@ -0,0 +1,155 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: create nokia directory + file: + path: /usr/libexec/nokia/tmp + state: directory + mode: 0700 + become: true + become_user: "root" + +- name: create the /etc/openssl directory + file: + path: "/etc/openssl" + state: directory + become: true + become_user: "root" + no_log: True + +- name: create /var/lib/docker directory + file: + path: "/var/lib/docker" + state: directory + become: true + become_user: "root" + +- name: create /srv/node/swift directory + file: + path: "/srv/node/swift" + state: directory + become_user: "root" + +- name: template enc file1 + copy: + content: "{{ caas.encrypted_ca[0] | replace(' ', '\n')}}" + dest: /etc/openssl/ca.pem.enc + become_user: "root" + no_log: True + +- name: template enc file2 + copy: + content: "{{ caas.encrypted_ca_key[0] | replace(' ', '\n')}}" + dest: /etc/openssl/ca-key.pem.enc + become_user: "root" + no_log: True + +- name: template podruntime slice file + template: + src: podruntime.slice + dest: /usr/lib/systemd/system/podruntime.slice + become_user: "root" + +- name: adding acl for cloud_admin_user for /usr/bin/helm + acl: + entity: "{{ users.admin_user_name }}" + etype: user + path: "/usr/bin/helm" + permissions: "rx" + recursive: yes + state: present + become_user: "root" + +- name: adding acl for cloud_admin_user for /usr/bin/kubectl + acl: + entity: "{{ users.admin_user_name }}" + etype: user + path: "/usr/bin/kubectl" + permissions: "rx" + recursive: yes + state: present + become_user: "root" + +- name: adding acl for cloud_admin_user for /usr/libexec/nokia/merge_image.sh + acl: + entity: "{{ users.admin_user_name }}" + etype: user + path: "/usr/libexec/nokia/merge_image.sh" + permissions: "rx" + recursive: yes + state: present + become_user: "root" + +- name: adding acl for cloud_admin_user to read /var/log + acl: + entity: "{{ users.admin_user_name }}" + etype: user + path: "/var/log" + permissions: "rx" + recursive: yes + state: present + become_user: "root" + +- name: adding default for cloud_admin_user, /var/log + acl: + default: yes + entity: "{{ users.admin_user_name }}" + etype: user + path: "/var/log" + permissions: "rx" + recursive: yes + state: present + become_user: "root" + +- name: adding acl for cloud_admin_user to access caas_chart_path + acl: + entity: "{{ users.admin_user_name }}" + etype: user + path: "{{ caas.caas_chart_path }}" + permissions: "rwx" + recursive: yes + state: present + become_user: "root" + +- name: adding default for cloud_admin_user to access caas_chart_path + acl: + default: yes + entity: "{{ users.admin_user_name }}" + etype: user + path: "{{ caas.caas_chart_path }}" + permissions: "rwx" + recursive: yes + state: present + become_user: "root" + +- name: modify the kernel parameters for elasticsearch + shell: 'sysctl -w vm.max_map_count=262144' + when: nodetype | search('caas_master') + become_user: "root" + +- name: create /var/log/elasticsearch directory + file: + path: /var/log/elasticsearch + recurse: yes + owner: "{{ caas.uid.elasticsearch }}" + group: "{{ caas.uid.elasticsearch }}" + state: directory + become_user: "root" + when: (nodetype is defined) and (nodetype | search('caas_master')) + +- name: update bash-completion for kubernetes + shell: "kubectl completion bash > /etc/bash_completion.d/kubectl-bash" + become: true + become_user: "root" diff --git a/ansible/roles/pre_config_all/templates/podruntime.slice b/ansible/roles/pre_config_all/templates/podruntime.slice new file mode 100644 index 0000000..67d29af --- /dev/null +++ b/ansible/roles/pre_config_all/templates/podruntime.slice @@ -0,0 +1,22 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +[Unit] +Description=Limited resources slice for Kubernetes services +Documentation=man:systemd.special(7) +DefaultDependencies=no +Before=slices.target +Requires=-.slice +After=-.slice diff --git a/ansible/roles/pre_install_caas_infra/meta/main.yaml b/ansible/roles/pre_install_caas_infra/meta/main.yaml new file mode 100644 index 0000000..7506305 --- /dev/null +++ b/ansible/roles/pre_install_caas_infra/meta/main.yaml @@ -0,0 +1,52 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - role: cert + instance: "cpuwebhook" + cert_name: "cpuwebhook.crt" + key_name: "cpuwebhook.key" + cert_path: /etc/kubernetes/ssl + alt_names: + dns: + - "cpu-dev-pod-mutator-svc.kube-system.svc.nokia.net" + - "cpu-dev-pod-mutator-svc.kube-system.svc" + become: true + become_user: "root" + + - role: cert + instance: "prometheus" + cert_name: "prometheus.crt" + key_name: "prometheus.key" + common_name: "prometheus" + cert_path: /etc/kubernetes/ssl + alt_names: + dns: + - "{{ caas.prometheus_url }}" + become: true + become_user: "root" + + - role: cert + instance: "metrics-server" + cert_name: "metrics-server.crt" + key_name: "metrics-server.key" + common_name: "metrics-server" + cert_path: /etc/metrics-server/ssl + alt_names: + dns: + - metrics-server.kube-system.svc.nokia.net + - metrics-server.kube-system.svc + become: true + become_user: "root" diff --git a/ansible/roles/pre_install_caas_infra/tasks/main.yaml b/ansible/roles/pre_install_caas_infra/tasks/main.yaml new file mode 100644 index 0000000..4eef782 --- /dev/null +++ b/ansible/roles/pre_install_caas_infra/tasks/main.yaml @@ -0,0 +1,20 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + - name: prepare cpupooler + import_tasks: prepare_cpupooler.yaml + + - name: prepare metrics + import_tasks: prepare_metrics.yaml diff --git a/ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml b/ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml new file mode 100644 index 0000000..5bfac33 --- /dev/null +++ b/ansible/roles/pre_install_caas_infra/tasks/prepare_cpupooler.yaml @@ -0,0 +1,25 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: get ca bundle + shell: "cat /etc/openssl/ca.pem | base64 | tr -d '\n'" + register: temp_webhook_ca_bundle + become_user: "root" + no_log: true + +- name: set cpu_ca_bundle + set_fact: + webhooks_ca: "{{ temp_webhook_ca_bundle.stdout }}" + no_log: true diff --git a/ansible/roles/pre_install_caas_infra/tasks/prepare_metrics.yaml b/ansible/roles/pre_install_caas_infra/tasks/prepare_metrics.yaml new file mode 100644 index 0000000..64b0dae --- /dev/null +++ b/ansible/roles/pre_install_caas_infra/tasks/prepare_metrics.yaml @@ -0,0 +1,50 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: read metrics cert as base64 string + shell: "cat /etc/kubernetes/ssl/metrics.crt | base64 -w 0" + register: metrics_cert_content + +- name: read metrics cert key as base64 string + shell: "cat /etc/kubernetes/ssl/metrics.key | base64 -w 0" + register: metrics_cert_key_content + +- name: set b64 metrics cert variable + set_fact: + metrics_cert_b64: "{{ metrics_cert_content.stdout }}" + metrics_cert_key_b64: "{{ metrics_cert_key_content.stdout }}" + +- name: read prometheus cert as base64 string + shell: "cat /etc/kubernetes/ssl/prometheus.crt | base64 -w 0" + register: prometheus_cert_content + become_user: "root" + +- name: read prometheus cert key as base64 string + shell: "cat /etc/kubernetes/ssl/prometheus.key | base64 -w 0" + register: prometheus_cert_key_content + become_user: "root" + +- name: set b64 prometheus cert variable + set_fact: + prometheus_cert_b64: "{{ prometheus_cert_content.stdout }}" + prometheus_cert_key_b64: "{{ prometheus_cert_key_content.stdout }}" + +- name: remove prometheus key and cert + file: path='{{ item }}' state=absent + with_items: + - /etc/kubernetes/ssl/prometheus.key + - /etc/kubernetes/ssl/prometheus.crt + become_user: "root" + diff --git a/cm_config/caas.yaml b/cm_config/caas.yaml new file mode 100644 index 0000000..7b86625 --- /dev/null +++ b/cm_config/caas.yaml @@ -0,0 +1,102 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +salt: malaise68Adroitness44Gradate45Probe41Settle46Newsweek93restrict8Thirsty35abused9Pluperfect +max_login_time: 30 +max_parallel_session: 15 +ansible_success: 'Ansible deploy END success' +docker_directory: /var/lib/docker +cert_directory: /etc/docker +caas_base_directory: "/var/lib/caas" +infra_containers_directory: "{{ caas_base_directory }}/images" +manifests_directory: "{{ caas_base_directory }}/manifests" +registry_directory: /var/lib/registry +apiserver_in_hosts: apiserver.nokia.net +etcd_api_port: 4111 +etcd_peer_port: 4112 +kubernetes_component_label: ncrf-image +swift_url: swift.kube-system.svc.nokia.net +swift_port: 8084 +swift: https://{{ swift_url }}:{{ swift_port }} +swift_update_url: swift-update.kube-system.svc.nokia.net +swift_memcached_port: 11222 +update_swift_port: 8091 +swift_update: https://{{ swift_update_url }}:{{ update_swift_port }} +chart_repo_ip: 10.254.210.229 +chart_repo_url: chart-repo.kube-system.svc.nokia.net +chart_repo_port: 8088 +chart_repo: https://{{ chart_repo_url }}:{{ chart_repo_port }} +tiller_ip: 10.254.165.237 +tiller_url: tiller.kube-system.svc.nokia.net +tiller_port: 44134 +tiller: https://{{ tiller_url }}:{{ tiller_port }} +registry_url: registry.kube-system.svc.nokia.net +registry_port: 5555 +registry: https://{{ registry_url }}:{{ registry_port }} +registry_repo: "caas" +update_registry_url: registry-update.kube-system.svc.nokia.net +update_registry_port: 5556 +update_registry: https://{{ update_registry_url }}:{{ update_registry_port }} +elasticsearch_url: elasticsearch-logging.kube-system.svc.nokia.net +elasticsearch_port: 9200 +elasticsearch_memory_request: 1Gi +elasticsearch_memory_limit: 2Gi +elasticsearch_java_heap_request: 800m +elasticsearch_java_heap_max: 800m +elasticsearch_port_for_cluster_communication: 9300 +elasticsearch: https://{{ elasticsearch_url }}:{{ elasticsearch_port }} +internal_flannel_interface: eth0 +container_wait_timeout: 120 +ldap_master_url: master.infraldap-server.nokia.net +ldap_slave_url: infraldap-server-slave.kube-system.svc.nokia.net +apiserver_secure_port: 6443 +apiserver_svc_port: 443 +default_partition_size: 16 +danmnet_default_name: flannel +danmnet_default_namespace: kube-system +danmnet_default_network_id: flannel +danmnet_default_network_type: flannel +caas_chart_path: "{{ caas_base_directory }}/infra-charts" +caas_policy_directory: "{{ caas_base_directory }}/policies" +kubernetes_service_url: kubernetes.default.svc +prometheus_port: 9090 +prometheus_url: prometheus.kube-system.svc.nokia.net +prometheus: https://{{ prometheus_url }}:{{ prometheus_port }} +custom_metrics_api_port: 6443 +custom_metrics_api_serviceport: 443 +metrics_server_port: 443 +token_filename: token +tokenscsv_filename: tokens.csv +cert_path: /etc/kubernetes/ssl +_secrets_conf: secrets.conf +kubernetes_secret_users: kube +service_cluster_ip_cidr: "10.254.0.0/16" +apiserver_svc_ip: "10.254.0.1" +dns_svc_ip: "10.254.0.254" +apiserver_service_name: "kubernetes.default.svc.nokia.net" +rbac_manifests_directory: "{{ caas_base_directory }}/rbac_manifests" +webhook_port: 443 +root_partition: "/dev/vda" +uid: + docker: 142 + caas_etcd: 143 + kube: 144 + kubedns: 145 + swift: 146 + danm: 147 + dockerreg: 149 + chartrepo: 150 + elasticsearch: 202 + flannel: 666 diff --git a/infra-charts/Chart.yaml b/infra-charts/Chart.yaml new file mode 100644 index 0000000..4ca0ba7 --- /dev/null +++ b/infra-charts/Chart.yaml @@ -0,0 +1,22 @@ +--- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: caas-infra +version: 1.0.0 +description: CaaS infra package +maintainers: + - name: Balint Varga + email: balint.varga@nokia.com + diff --git a/infra-charts/templates/cm-adapter-serving-certs.yaml b/infra-charts/templates/cm-adapter-serving-certs.yaml new file mode 100644 index 0000000..bbf3ca4 --- /dev/null +++ b/infra-charts/templates/cm-adapter-serving-certs.yaml @@ -0,0 +1,26 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.custom_metrics_api.required }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cm-adapter-serving-certs + namespace: kube-system +data: + metrics.crt: {{ .Values.custom_metrics_api.server_cert }} + metrics.key: {{ .Values.custom_metrics_api.server_key }} +{{ end }} diff --git a/infra-charts/templates/cpu-deviceplugin-daemonset.yaml b/infra-charts/templates/cpu-deviceplugin-daemonset.yaml new file mode 100644 index 0000000..f150d4d --- /dev/null +++ b/infra-charts/templates/cpu-deviceplugin-daemonset.yaml @@ -0,0 +1,66 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.cpupooler.required }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cpu-device-plugin + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: cpu-device-plugin +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: cpu-device-plugin + template: + metadata: + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + labels: + {{ .Values.kubernetes_component_label }}: cpu-device-plugin + spec: + containers: + - name: cpu-device-plugin + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/cpupooler:{{ .Values.cpupooler.container_version }} + imagePullPolicy: IfNotPresent + command: [ "/cpu-device-plugin", "-logtostderr" ] + volumeMounts: + - mountPath: /etc/cpu-pooler + name: cpu-pooler-config + - mountPath: /var/lib/kubelet/device-plugins/ + name: devicesock + readOnly: false + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: devicesock + hostPath: + # directory location on host + path: /var/lib/kubelet/device-plugins/ + - name: cpu-pooler-config + configMap: + name: cpu-pooler-configmap + serviceAccountName: cpu-device-plugin +{{ end }} diff --git a/infra-charts/templates/cpu-pooler-configmap.yaml b/infra-charts/templates/cpu-pooler-configmap.yaml new file mode 100644 index 0000000..6966ca2 --- /dev/null +++ b/infra-charts/templates/cpu-pooler-configmap.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.cpupooler.required }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cpu-pooler-configmap + namespace: kube-system +data: +{{ range $hostname, $poolconfig := .Values.cpupooler.cpu_poolconfigs }} + poolconfig-{{ $hostname }}.yaml: | +{{ $poolconfig | toYaml | indent 4 }} +{{- end }} +{{ end }} diff --git a/infra-charts/templates/cpu-setter-daemonset.yaml b/infra-charts/templates/cpu-setter-daemonset.yaml new file mode 100644 index 0000000..6348525 --- /dev/null +++ b/infra-charts/templates/cpu-setter-daemonset.yaml @@ -0,0 +1,65 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.cpupooler.required }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cpu-setter + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: cpu-setter +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: cpu-setter + template: + metadata: + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + labels: + {{ .Values.kubernetes_component_label }}: cpu-setter + spec: + containers: + - name: cpu-device-plugin + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/cpupooler:{{ .Values.cpupooler.container_version }} + imagePullPolicy: IfNotPresent + command: [ "/cpu-setter", "--poolconfigs=/etc/cpu-pooler", "--cpusetroot=/sys/fs/cgroup/cpuset/kubepods/" ] + volumeMounts: + - mountPath: /etc/cpu-pooler + name: cpu-pooler-config + - mountPath: /var/lib/kubelet/device-plugins/kubelet_internal_checkpoint + name: checkpointfile + readOnly: false + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: checkpointfile + hostPath: + path: /var/lib/kubelet/device-plugins/kubelet_internal_checkpoint + - name: cpu-pooler-config + configMap: + name: cpu-pooler-configmap + serviceAccountName: cpu-setter +{{ end }} diff --git a/infra-charts/templates/custom-metrics-apiserver-config-map.yaml b/infra-charts/templates/custom-metrics-apiserver-config-map.yaml new file mode 100644 index 0000000..205b014 --- /dev/null +++ b/infra-charts/templates/custom-metrics-apiserver-config-map.yaml @@ -0,0 +1,92 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.custom_metrics_api.required }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: adapter-config + namespace: kube-system +data: + config.yaml: | + rules: + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{ end }} diff --git a/infra-charts/templates/custom-metrics-apiserver-deployment.yaml b/infra-charts/templates/custom-metrics-apiserver-deployment.yaml new file mode 100755 index 0000000..7fca746 --- /dev/null +++ b/infra-charts/templates/custom-metrics-apiserver-deployment.yaml @@ -0,0 +1,94 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.custom_metrics_api.required }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + {{ .Values.kubernetes_component_label }}: custom-metrics-apiserver + name: custom-metrics-apiserver + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: custom-metrics-apiserver + template: + metadata: + labels: + {{ .Values.kubernetes_component_label }}: custom-metrics-apiserver + name: custom-metrics-apiserver + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + spec: + nodeSelector: + nodetype: caas_master + serviceAccountName: custom-metrics-apiserver + containers: + - name: custom-metrics-apiserver + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/custom_metrics:{{ .Values.custom_metrics_api.container_version }} + args: + - /adapter + - --secure-port={{ .Values.custom_metrics_api.port }} + - --tls-cert-file=/var/run/serving-cert/metrics.crt + - --tls-private-key-file=/var/run/serving-cert/metrics.key + - --logtostderr=true + - --prometheus-url={{ .Values.prometheus.prometheus_url }} + - --metrics-relist-interval=30s + - --discovery-interval=5m + - --v=1 + - --config=/etc/adapter/config.yaml + ports: + - containerPort: {{ .Values.custom_metrics_api.port }} + volumeMounts: + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + - name: secrets + mountPath: /etc/kubernetes/ssl + readOnly: true + - name: secret-root-ca + mountPath: /etc/ssl/certs/ca.pem + readOnly: true + - name: auth + mountPath: /config + readOnly: true + - mountPath: /etc/adapter/ + name: config + readOnly: true + volumes: + - name: volume-serving-cert + secret: + secretName: cm-adapter-serving-certs + - name: secrets + hostPath: + path: /etc/kubernetes/ssl + - name: secret-root-ca + hostPath: + path: /etc/openssl/ca.pem + - name: auth + hostPath: + path: /etc/kubernetes/kubeconfig + - name: config + configMap: + name: adapter-config +{{ end }} diff --git a/infra-charts/templates/custom-metrics-apiserver-service.yaml b/infra-charts/templates/custom-metrics-apiserver-service.yaml new file mode 100755 index 0000000..a92f1be --- /dev/null +++ b/infra-charts/templates/custom-metrics-apiserver-service.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.custom_metrics_api.required }} +--- +apiVersion: v1 +kind: Service +metadata: + name: custom-metrics-apiserver + namespace: kube-system +spec: + ports: + - port: {{ .Values.custom_metrics_api.serviceport }} + targetPort: {{ .Values.custom_metrics_api.port }} + selector: + {{ .Values.kubernetes_component_label }}: custom-metrics-apiserver +{{ end }} diff --git a/infra-charts/templates/custom-metrics-apiservice.yaml b/infra-charts/templates/custom-metrics-apiservice.yaml new file mode 100755 index 0000000..05bb88e --- /dev/null +++ b/infra-charts/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.custom_metrics_api.required }} +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: custom-metrics-apiserver + namespace: kube-system + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +{{ end }} diff --git a/infra-charts/templates/elasticsearch-configmap.yaml b/infra-charts/templates/elasticsearch-configmap.yaml new file mode 100644 index 0000000..3e78938 --- /dev/null +++ b/infra-charts/templates/elasticsearch-configmap.yaml @@ -0,0 +1,149 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.elasticsearch.required }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch + namespace: kube-system + labels: + app: elasticsearch +data: + elasticsearch.yml: |- + cluster.name: {{ .Values.elasticsearch.cluster.name }} + + node.data: ${NODE_DATA:true} + node.master: ${NODE_MASTER:true} + node.ingest: ${NODE_INGEST:true} + node.name: ${HOSTNAME} + + network.host: 0.0.0.0 + bootstrap.memory_lock: ${BOOTSTRAP_MEMORY_LOCK:false} + + discovery: + zen: + ping.unicast.hosts: ${DISCOVERY_SERVICE:} + minimum_master_nodes: ${MINIMUM_MASTER_NODES:2} + + processors: ${PROCESSORS:} + + # avoid split-brain w/ a minimum consensus of 3 masters plus 3 data nodes + gateway.expected_master_nodes: ${EXPECTED_MASTER_NODES:3} + gateway.expected_data_nodes: ${EXPECTED_DATA_NODES:3} + gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m} + gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:3} + gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:3} + + log4j2.properties: |- + status = error + appender.console.type = Console + appender.console.name = console + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + rootLogger.level = info + rootLogger.appenderRef.console.ref = console + logger.searchguard.name = com.floragunn + logger.searchguard.level = info + + supervisord.conf: | + [unix_http_server] + file=/run/supervisord.sock ; (the path to the socket file) + chmod = 0770 + chown = root:root + username = dummy + password = dummy + + ;[inet_http_server] ; inet (TCP) server disabled by default + + [supervisord] + user=root + logfile=/dev/null + logfile_maxbytes=0 + loglevel=info + pidfile=/var/run/supervisor/supervisord.pid + nodaemon=true + minfds=1024 + minprocs=200 + + [supervisorctl] + serverurl=unix:///run/supervisord.sock + username = dummy + password = dummy + + [rpcinterface:supervisor] + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + [program:elasticsearch] + command=/usr/share/elasticsearch/bin/elasticsearch + user= {{ .Values.elasticsearch.es_uid }} + startsecs=3 + startretries=1 + stopwaitsecs = 3 + priority = 1 + redirect_stderr=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + + [program:crond] + command=/usr/sbin/crond -n -s + startsecs=3 + startretries=1 + stopwaitsecs = 3 + priority = 2 + redirect_stderr=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + + [program:setindex] + command=/usr/share/elasticsearch/elasticsearch-setindex + startsecs=3 + startretries=1 + stopwaitsecs = 3 + priority = 2 + redirect_stderr=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + + elasticsearch-logrotate: | + #!/bin/bash + # Copyright 2019 Nokia + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + FS_LIMIT=80 + + declare -i pcent=100 + + + while [ $pcent -ge $FS_LIMIT ] + do + pcent=`df --output=pcent /usr/share/elasticsearch/data | tail -n1 | tr -d '%'` + if [ $pcent -le $FS_LIMIT ]; then exit; fi + oldest=`curl -s -XGET 'localhost:9200/_cat/indices' | cut -d' ' -f3 | grep ^logstash | sort | head -n1` + curl -XDELETE "localhost:9200/$oldest" >/dev/null + done +{{ end }} diff --git a/infra-charts/templates/elasticsearch-data-statefulset.yaml b/infra-charts/templates/elasticsearch-data-statefulset.yaml new file mode 100644 index 0000000..246be09 --- /dev/null +++ b/infra-charts/templates/elasticsearch-data-statefulset.yaml @@ -0,0 +1,122 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.elasticsearch.required }} +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + labels: + app: elasticsearch-data + component: "{{ .Values.elasticsearch.data.name }}" + name: elasticsearch-data + namespace: kube-system +spec: + serviceName: elasticsearch-data + replicas: {{ .Values.elasticsearch.data.replicas }} + template: + metadata: + labels: + app: elasticsearch-data + component: "{{ .Values.elasticsearch.data.name }}" + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + app: elasticsearch-data + component: "{{ .Values.elasticsearch.data.name }}" + nodeSelector: + nodetype: caas_master + containers: + - name: elasticsearch-data + env: + - name: NODE_MASTER + value: "false" + - name: DISCOVERY_SERVICE + value: elasticsearch-data.kube-system.svc.nokia.net + - name: PROCESSORS + valueFrom: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS + value: "-Xms{{ .Values.elasticsearch.java_heap_request }} -Xmx{{ .Values.elasticsearch.java_heap_max }} -Dmapper.allow_dots_in_name=true" + {{- range $key, $value := .Values.elasticsearch.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + image: {{ .Values.elasticsearch.image_name }} + command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"] + ports: + - containerPort: {{ .Values.elasticsearch.port_for_cluster_communication }} + name: transport + resources: + limits: + cpu: "{{ .Values.elasticsearch.cpu_limit }}" + memory: "{{ .Values.elasticsearch.memory_limit }}" + requests: + memory: "{{ .Values.elasticsearch.memory_request }}" + readinessProbe: + tcpSocket: + port: {{ .Values.elasticsearch.port }} + initialDelaySeconds: 5 + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: data + - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + name: elasticsearch-config + subPath: elasticsearch.yml + - mountPath: /etc/supervisord.conf + name: supervisord-config + subPath: supervisord.conf + - mountPath: /etc/cron.hourly/elasticsearch-logrotate + name: elasticsearch-logrotate + subPath: elasticsearch-logrotate + terminationGracePeriodSeconds: {{ .Values.elasticsearch.data.terminationGracePeriodSeconds }} + volumes: + - name: elasticsearch-config + configMap: + name: elasticsearch + items: + - key: elasticsearch.yml + path: elasticsearch.yml + mode: 0644 + - name: supervisord-config + configMap: + name: elasticsearch + items: + - key: supervisord.conf + path: supervisord.conf + mode: 0644 + - name: elasticsearch-logrotate + configMap: + name: elasticsearch + items: + - key: elasticsearch-logrotate + path: elasticsearch-logrotate + mode: 0755 + - name: data + hostPath: + path: /var/log/elasticsearch +{{ end }} diff --git a/infra-charts/templates/elasticsearch-master-statefulset.yaml b/infra-charts/templates/elasticsearch-master-statefulset.yaml new file mode 100644 index 0000000..50c0c8f --- /dev/null +++ b/infra-charts/templates/elasticsearch-master-statefulset.yaml @@ -0,0 +1,108 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.elasticsearch.required }} +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + labels: + app: elasticsearch-master + component: "{{ .Values.elasticsearch.master.name }}" + name: elasticsearch-master + namespace: kube-system +spec: + serviceName: elasticsearch-master + replicas: {{ .Values.elasticsearch.master.replicas }} + template: + metadata: + labels: + app: elasticsearch-master + component: "{{ .Values.elasticsearch.master.name }}" + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + app: elasticsearch-master + component: "{{ .Values.elasticsearch.master.name }}" + nodeSelector: + nodetype: caas_master + containers: + - name: elasticsearch-master + env: + - name: NODE_DATA + value: "false" + - name: DISCOVERY_SERVICE + value: elasticsearch-data.kube-system.svc.nokia.net + - name: ES_JAVA_OPTS + value: "-Xms{{ .Values.elasticsearch.java_heap_request }} -Xmx{{ .Values.elasticsearch.java_heap_max }} -Dmapper.allow_dots_in_name=true" + {{- range $key, $value := .Values.elasticsearch.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: + limits: + memory: "{{ .Values.elasticsearch.memory_limit }}" + requests: + memory: "{{ .Values.elasticsearch.memory_request }}" + readinessProbe: + httpGet: + path: /_cluster/health?local=true + port: {{ .Values.elasticsearch.port }} + initialDelaySeconds: 5 + image: {{ .Values.elasticsearch.image_name }} + command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"] + ports: + - containerPort: {{ .Values.elasticsearch.port_for_cluster_communication }} + name: transport + - containerPort: {{ .Values.elasticsearch.port }} + name: http + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: data + - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + name: elasticsearch-config + subPath: elasticsearch.yml + - mountPath: /etc/supervisord.conf + name: supervisord-config + subPath: supervisord.conf + volumes: + - name: elasticsearch-config + configMap: + name: elasticsearch + items: + - key: elasticsearch.yml + path: elasticsearch.yml + mode: 0644 + - name: supervisord-config + configMap: + name: elasticsearch + items: + - key: supervisord.conf + path: supervisord.conf + mode: 0644 + - name: data + emptyDir: {} +{{ end }} diff --git a/infra-charts/templates/elasticsearch-service.yaml b/infra-charts/templates/elasticsearch-service.yaml new file mode 100644 index 0000000..5e70e20 --- /dev/null +++ b/infra-charts/templates/elasticsearch-service.yaml @@ -0,0 +1,49 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.elasticsearch.required }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: elasticsearch-data + component: "{{ .Values.elasticsearch.data.name }}" + name: elasticsearch-data + namespace: kube-system +spec: + ports: + - port: {{ .Values.elasticsearch.port_for_cluster_communication }} + targetPort: transport + selector: + app: elasticsearch-data + component: "{{ .Values.elasticsearch.data.name }}" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: elasticsearch-master + component: "{{ .Values.elasticsearch.master.name }}" + name: elasticsearch-logging + namespace: kube-system +spec: + ports: + - port: {{ .Values.elasticsearch.port }} + targetPort: http + selector: + app: elasticsearch-master + component: "{{ .Values.elasticsearch.master.name }}" +{{ end }} diff --git a/infra-charts/templates/metrics-apiservice.yaml b/infra-charts/templates/metrics-apiservice.yaml new file mode 100755 index 0000000..c2486f3 --- /dev/null +++ b/infra-charts/templates/metrics-apiservice.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.metrics_server.required }} +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +{{ end }} diff --git a/infra-charts/templates/metrics-server-deployment.yaml b/infra-charts/templates/metrics-server-deployment.yaml new file mode 100755 index 0000000..71d2a0e --- /dev/null +++ b/infra-charts/templates/metrics-server-deployment.yaml @@ -0,0 +1,61 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.metrics_server.required }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: metrics-server +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: metrics-server + template: + metadata: + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + name: metrics-server + labels: + {{ .Values.kubernetes_component_label }}: metrics-server + spec: + nodeSelector: + nodetype: caas_master + serviceAccountName: metrics-apiserver + containers: + - name: metrics-server + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/metrics_server:{{ .Values.metrics_server.container_version }} + command: + - /metrics-server + - "--logtostderr" + - "--tls-cert-file=/etc/metrics-server/ssl/metrics-server.crt" + - "--tls-private-key-file=/etc/metrics-server/ssl/metrics-server.key" + volumeMounts: + - name: stuffs + mountPath: /etc/metrics-server + readOnly: true + volumes: + - name: stuffs + hostPath: + path: /etc/metrics-server +{{ end }} diff --git a/infra-charts/templates/metrics-server-service.yaml b/infra-charts/templates/metrics-server-service.yaml new file mode 100755 index 0000000..d939538 --- /dev/null +++ b/infra-charts/templates/metrics-server-service.yaml @@ -0,0 +1,32 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.metrics_server.required }} +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" +spec: + selector: + {{ .Values.kubernetes_component_label }}: metrics-server + ports: + - port: {{ .Values.metrics_server.port }} + protocol: TCP + targetPort: {{ .Values.metrics_server.port }} +{{ end }} diff --git a/infra-charts/templates/netwatcher.yaml b/infra-charts/templates/netwatcher.yaml new file mode 100644 index 0000000..5fa4723 --- /dev/null +++ b/infra-charts/templates/netwatcher.yaml @@ -0,0 +1,49 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: netwatcher + namespace: kube-system +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: netwatcher + template: + metadata: + labels: + {{ .Values.kubernetes_component_label }}: netwatcher + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: netwatcher + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/netwatcher:{{ .Values.netwatcher.container_version }} + securityContext: + capabilities: + add: + - SYS_PTRACE + - SYS_ADMIN + - NET_ADMIN + - NET_RAW + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + terminationGracePeriodSeconds: 0 + serviceAccountName: netwatcher diff --git a/infra-charts/templates/prometheus-cfg.yaml b/infra-charts/templates/prometheus-cfg.yaml new file mode 100644 index 0000000..7a8b559 --- /dev/null +++ b/infra-charts/templates/prometheus-cfg.yaml @@ -0,0 +1,321 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.prometheus.required }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + {{ .Values.kubernetes_component_label }}: prometheus + name: prometheus-config + namespace: kube-system +data: + prometheus.yml: | + # A scrape configuration for running Prometheus on a Kubernetes cluster. + # This uses separate scrape configs for cluster components (i.e. API server, node) + # and services to allow each to use different authentication configs. + # + # Kubernetes labels will be added as Prometheus labels on metrics via the + # `labelmap` relabeling action. + # + # If you are using Kubernetes 1.7.2 or earlier, please take note of the comments + # for the kubernetes-cadvisor job; you will need to edit or remove this job. + + # Scrape config for API servers. + # + # Kubernetes exposes API servers as endpoints to the default/kubernetes + # service so this uses `endpoints` role and uses relabelling to only keep + # the endpoints associated with the default/kubernetes service using the + # default named port `https`. This works for single API server deployments as + # well as HA API server deployments. + global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 1m + + scrape_configs: + - job_name: 'kubernetes-apiservers' + + kubernetes_sd_configs: + - role: endpoints + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # If your node certificates are self-signed or use a different CA to the + # master CA, then disable certificate verification below. Note that + # certificate verification is an integral part of a secure infrastructure + # so this should only be disabled in a controlled environment. You can + # disable certificate verification by uncommenting the line below. + # + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + # Keep only the default/kubernetes service endpoints for the https port. This + # will add targets for each API server which Kubernetes adds an endpoint to + # the default/kubernetes service. + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + # Scrape config for nodes (kubelet). + # + # Rather than connecting directly to the node, the scrape is proxied though the + # Kubernetes apiserver. This means it will work if Prometheus is running out of + # cluster, or can't connect to nodes for some other reason (e.g. because of + # firewalling). + - job_name: 'kubernetes-nodes' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # Scrape config for Kubelet cAdvisor. + # + # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics + # (those whose names begin with 'container_') have been removed from the + # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to + # retrieve those metrics. + # + # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor + # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" + # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with + # the --cadvisor-port=0 Kubelet flag). + # + # This job is not necessary and should be removed in Kubernetes 1.6 and + # earlier versions, or it will cause the metrics to be scraped twice. + - job_name: 'kubernetes-cadvisor' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # Scrape config for service endpoints. + # + # The relabeling allows the actual service scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/scrape`: Only scrape services that have a value of `true` + # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need + # to set this to `https` & most likely set the `tls_config` of the scrape config. + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: If the metrics are exposed on a different port to the + # service then set this appropriately. + - job_name: 'kubernetes-service-endpoints' + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + # Example scrape config for pods + # + # The relabeling allows the actual pod scrape endpoint to be configured via the + # following annotations: + # + # * `prometheus.io/scrape`: Only scrape pods that have a value of `true` + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the + # pod's declared ports (default is a port-free target if none are declared). + - job_name: 'kubernetes-pods' + # if you want to use metrics on jobs, set the below field to + # true to prevent Prometheus from setting the `job` label + # automatically. + honor_labels: false + kubernetes_sd_configs: + - role: pod + # skip verification so you can do HTTPS to pods + tls_config: + insecure_skip_verify: true + # make sure your labels are in order + relabel_configs: + # these labels tell Prometheus to automatically attach source + # pod and namespace information to each collected sample, so + # that they'll be exposed in the custom metrics API automatically. + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod + # these labels tell Prometheus to look for + # prometheus.io/{scrape,path,port} annotations to configure + # how to scrape + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (.+) + nginx.conf: | + daemon off; + worker_processes auto; + error_log /dev/stderr; + pid /run/nginx.pid; + include /etc/nginx/modules/*.conf; + events { + worker_connections 1024; + } + http { + server { + listen {{ .Values.prometheus.port }} ssl; + ssl_certificate /var/run/serving-cert/prometheus.crt; + ssl_certificate_key /var/run/serving-cert/prometheus.key; + ssl_client_certificate /etc/openssl/ca.pem; + ssl_verify_client optional; + location / { + proxy_pass http://127.0.0.1:19090/; + } + } + } + supervisord.conf: | + [unix_http_server] + file=/run/supervisord.sock ; (the path to the socket file) + chmod = 0770 + chown = root:root + username = dummy + password = dummy + + ;[inet_http_server] ; inet (TCP) server disabled by default + + [supervisord] + user=root + logfile=/dev/null + logfile_maxbytes=0 + loglevel=info + pidfile=/var/run/supervisor/supervisord.pid + nodaemon=true + minfds=1024 + minprocs=200 + + [supervisorctl] + serverurl=unix:///run/supervisord.sock + username = dummy + password = dummy + + [rpcinterface:supervisor] + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + [program:prometheus] + command=/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.retention=1h --web.listen-address=127.0.0.1:19090 + startsecs=3 + startretries=1 + stopwaitsecs = 3 + priority = 1 + redirect_stderr=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + + [program:nginx] + command=/usr/sbin/nginx -c /etc/prometheus/nginx.conf + startsecs=3 + startretries=1 + stopwaitsecs = 3 + priority = 2 + redirect_stderr=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 +{{ end }} diff --git a/infra-charts/templates/prometheus-dep.yaml b/infra-charts/templates/prometheus-dep.yaml new file mode 100644 index 0000000..1d260a3 --- /dev/null +++ b/infra-charts/templates/prometheus-dep.yaml @@ -0,0 +1,94 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.prometheus.required }} +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: prometheus + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: prometheus + template: + metadata: + labels: + {{ .Values.kubernetes_component_label }}: prometheus + annotations: + prometheus.io/scrape: 'false' + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + spec: + nodeSelector: + nodetype: caas_master + serviceAccountName: prometheus + containers: + - name: prometheus + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/prometheus:{{ .Values.prometheus.container_version }} + command: + - /usr/bin/supervisord + args: + - -c + - /etc/prometheus/supervisord.conf + ports: + - containerPort: {{ .Values.prometheus.port }} + protocol: TCP + resources: + limits: + memory: 2Gi + volumeMounts: + - mountPath: /var/run/serving-cert + name: prometheus-serving-cert + readOnly: true + - name: secret-root-ca + mountPath: /etc/openssl/ca.pem + readOnly: true + - mountPath: /etc/prometheus + name: prometheus-config + livenessProbe: + httpGet: + path: /api/v1/status/config + port: {{ .Values.prometheus.port }} + scheme: HTTPS + initialDelaySeconds: 3 + timeoutSeconds: 5 + volumes: + - name: prometheus-serving-cert + secret: + secretName: prometheus-serving-certs + - name: secret-root-ca + hostPath: + path: /etc/openssl/ca.pem + - name: prometheus-config + configMap: + name: prometheus-config + items: + - key: prometheus.yml + path: prometheus.yml + mode: 0644 + - key: nginx.conf + path: nginx.conf + mode: 0644 + - key: supervisord.conf + path: supervisord.conf + mode: 0644 +{{ end }} diff --git a/infra-charts/templates/prometheus-serving-certs.yaml b/infra-charts/templates/prometheus-serving-certs.yaml new file mode 100644 index 0000000..23e109b --- /dev/null +++ b/infra-charts/templates/prometheus-serving-certs.yaml @@ -0,0 +1,26 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.prometheus.required }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: prometheus-serving-certs + namespace: kube-system +data: + prometheus.crt: {{ .Values.prometheus.server_cert }} + prometheus.key: {{ .Values.prometheus.server_key }} +{{ end }} diff --git a/infra-charts/templates/prometheus-svc.yaml b/infra-charts/templates/prometheus-svc.yaml new file mode 100644 index 0000000..0e0cad0 --- /dev/null +++ b/infra-charts/templates/prometheus-svc.yaml @@ -0,0 +1,32 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.prometheus.required }} +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: prometheus +spec: + ports: + - port: {{ .Values.prometheus.port }} + targetPort: {{ .Values.prometheus.port }} + protocol: TCP + selector: + {{ .Values.kubernetes_component_label }}: prometheus +{{ end }} diff --git a/infra-charts/templates/sriovdp.yml b/infra-charts/templates/sriovdp.yml new file mode 100644 index 0000000..2a2f164 --- /dev/null +++ b/infra-charts/templates/sriovdp.yml @@ -0,0 +1,71 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.sriovdp.required }} +--- +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: sriovdp-ds + namespace: kube-system + labels: + app: sriovdp-app +spec: + selector: + matchLabels: + name: sriovdp + template: + metadata: + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + labels: + name: sriovdp + spec: + nodeSelector: + sriov: enabled + containers: + - name: sriovdp + image: {{ .Values.sriovdp.image_name }} + args: + - "-logtostderr" + - "-v=10" + - "-resource-prefix=nokia.k8s.io" + volumeMounts: + - mountPath: /var/lib/kubelet/device-plugins/ + name: devicesock + readOnly: false + - mountPath: /sys/class/net/ + name: sysfs + readOnly: true + - name: sriovdp-config + mountPath: /etc/pcidp/ + readOnly: true + volumes: + - name: devicesock + hostPath: + path: /var/lib/kubelet/device-plugins/ + - name: sysfs + hostPath: + path: /sys/class/net/ + - name: sriovdp-config + hostPath: + path: /etc/pcidp/ + terminationGracePeriodSeconds: 1 +{{ end }} diff --git a/infra-charts/templates/svcwatcher.yaml b/infra-charts/templates/svcwatcher.yaml new file mode 100644 index 0000000..63676a6 --- /dev/null +++ b/infra-charts/templates/svcwatcher.yaml @@ -0,0 +1,54 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.svcwatcher.required }} +--- +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: svcwatcher + namespace: kube-system +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: svcwatcher + template: + metadata: + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + labels: + {{ .Values.kubernetes_component_label }}: svcwatcher + spec: + dnsPolicy: ClusterFirst + nodeSelector: + nodetype: caas_master + containers: + - name: svcwatcher + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/svcwatcher:{{ .Values.svcwatcher.container_version }} + args: + - "--logtostderr" + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + terminationGracePeriodSeconds: 0 + serviceAccountName: svcwatcher +{{ end }} diff --git a/infra-charts/templates/webhook-conf.yaml b/infra-charts/templates/webhook-conf.yaml new file mode 100644 index 0000000..611f590 --- /dev/null +++ b/infra-charts/templates/webhook-conf.yaml @@ -0,0 +1,40 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.cpupooler.required }} +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: cpu-device-pod-mutator-webhook-config + namespace: kube-system + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-delete-policy": hook-failed, before-hook-creation +webhooks: + - name: cpu-dev-mutator.nokia.k8s.io + clientConfig: + service: + name: cpu-dev-pod-mutator-svc + namespace: kube-system + path: "/mutating-pods" + caBundle: "{{ .Values.cpupooler.webhook_ca_bundle }}" + rules: + - operations: ["CREATE","UPDATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail +{{ end }} diff --git a/infra-charts/templates/webhook-deployment.yaml b/infra-charts/templates/webhook-deployment.yaml new file mode 100644 index 0000000..9e72c26 --- /dev/null +++ b/infra-charts/templates/webhook-deployment.yaml @@ -0,0 +1,63 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.cpupooler.required }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cpu-dev-pod-mutator-deployment + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator + template: + metadata: + annotations: + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + name: cpu-dev-pod-mutator + labels: + {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator + spec: + nodeSelector: + nodetype: caas_master + containers: + - name: cpu-dev-pod-mutator + image: {{ .Values.registry_url }}/{{ .Values.registry_reponame }}/cpupooler:{{ .Values.cpupooler.container_version }} + command: [ "/cpu-device-webhook", "-tls-cert-file=/etc/webhook/certs/cpuwebhook.crt", "-tls-private-key-file=/etc/webhook/certs/cpuwebhook.key", "-alsologtostderr", "-v=4" ] + imagePullPolicy: IfNotPresent + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + - name: cpu-pooler-config + mountPath: /etc/cpu-pooler + volumes: + # TODO: make it a secret + - name: webhook-certs + hostPath: + path: /etc/kubernetes/ssl/ + - name: cpu-pooler-config + configMap: + name: cpu-pooler-configmap +{{ end }} diff --git a/infra-charts/templates/webhook-service.yaml b/infra-charts/templates/webhook-service.yaml new file mode 100644 index 0000000..cf6a332 --- /dev/null +++ b/infra-charts/templates/webhook-service.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ if .Values.cpupooler.required }} +--- +apiVersion: v1 +kind: Service +metadata: + name: cpu-dev-pod-mutator-svc + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator +spec: + ports: + - name: webhook + port: {{ .Values.cpupooler.webhook_port }} + selector: + {{ .Values.kubernetes_component_label }}: cpu-dev-pod-mutator +{{ end }} diff --git a/infra-charts/values.yaml.j2 b/infra-charts/values.yaml.j2 new file mode 100644 index 0000000..c5da700 --- /dev/null +++ b/infra-charts/values.yaml.j2 @@ -0,0 +1,82 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +--- +kubernetes_component_label: {{ caas.kubernetes_component_label }} +registry_url: {{ caas.registry_url }}:{{ caas.registry_port }} +registry_reponame: "{{ caas.registry_repo }}" + +metrics_server: + required: true + port: {{ caas.metrics_server_port }} + container_version: {{ container_image_names | select('search', '/metrics_server') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + +prometheus: + required: true + port: {{ caas.prometheus_port }} + container_version: {{ container_image_names | select('search', '/prometheus') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + prometheus_url: {{ caas.prometheus }} + server_cert: {{ prometheus_cert_b64 }} + server_key: {{ prometheus_cert_key_b64 }} + +custom_metrics_api: + required: true + port: {{ caas.custom_metrics_api_port }} + serviceport: {{ caas.custom_metrics_api_serviceport }} + container_version: {{ container_image_names | select('search', '/custom_metrics') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + server_cert: {{ metrics_cert_b64 }} + server_key: {{ metrics_cert_key_b64 }} + +svcwatcher: + required: true + container_version: {{ container_image_names | select('search', '/svcwatcher') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + +netwatcher: + container_version: {{ container_image_names | select('search', '/netwatcher') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + +cpupooler: + required: true + container_version: {{ container_image_names | select('search', '/cpupooler') | list | last | regex_replace('.*:([\\w\\-_\\.]+)$', '\\1') }} + webhook_ca_bundle: {{ webhooks_ca }} + webhook_port: {{ caas.webhook_port }} + cpu_poolconfigs: + {{ ansible_local['ncir_cpu_pooler_config'] | default([]) | to_nice_yaml | indent(4) }} + +elasticsearch: + required: true + port: {{ caas.elasticsearch_port }} + image_name: {{ container_image_names | select('search', '/elasticsearch') | list | last }} + memory_limit: {{ caas.elasticsearch_memory_limit }} + cpu_limit: 1 + memory_request: {{ caas.elasticsearch_memory_request }} + java_heap_request: {{ caas.elasticsearch_java_heap_request }} + java_heap_max: {{ caas.elasticsearch_java_heap_max }} + port_for_cluster_communication: {{ caas.elasticsearch_port_for_cluster_communication }} + es_uid: {{ caas.uid.elasticsearch }} + cluster: + name: "elasticsearch" + env: + MINIMUM_MASTER_NODES: "3" + master: + name: elasticsearch-master + replicas: 3 + data: + name: elasticsearch-data + replicas: 3 + terminationGracePeriodSeconds: 3600 + +sriovdp: + required: true + image_name: {{ container_image_names | select('search', '/sriovdp') | list | last }} diff --git a/utils/common/aliases.sh b/utils/common/aliases.sh new file mode 100644 index 0000000..d5d5c78 --- /dev/null +++ b/utils/common/aliases.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Swift +function swiftauth { source /etc/swift/usr/admin/env_file; export SWIFT_AUTH_KEY=`curl --cacert /etc/swift/main/tls-proxy/ca.pem -XGET -i -H"X-Auth-User:${SWIFT_TENANT}:${SWIFT_USER}" -H"X-Auth-Key:${SWIFT_PASS}" https://swift.kube-system.svc.nokia.net:8084/auth/v1.0 | grep X-Auth-Token: | awk '{print $2;}'`; } +function swiftget { curl --cacert /etc/swift/main/tls-proxy/ca.pem -XGET -H"X-Auth-Token: ${SWIFT_AUTH_KEY}" https://swift.kube-system.svc.nokia.net:8084/v1.0/AUTH_admin${1}; } +function swiftput { curl --cacert /etc/swift/main/tls-proxy/ca.pem -XPUT -H"X-Auth-Token: ${SWIFT_AUTH_KEY}" -T ${1} https://swift.kube-system.svc.nokia.net:8084/v1.0/AUTH_admin${2}; } diff --git a/utils/deploy/merge_image.sh b/utils/deploy/merge_image.sh new file mode 100644 index 0000000..d6f6c57 --- /dev/null +++ b/utils/deploy/merge_image.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /etc/profile + +CONC_MAX=4 + +conc() { + local procs=(`jobs -p`) + local proc_count=${#procs[*]} + + # Block until there is an open slot + if ((proc_count >= CONC_MAX)); then + wait ${procs[0]} + fi + + # Start our task + (eval "$@") & +} + +merge() { + TEMP=`getopt -o f:t:i: --long from-registry:,to-registry:,image-name: -- "$@"` + eval set -- "$TEMP" + + while [ -n "$1" ] + do + case "$1" in + -f|--from-registry) from_registry=$2; shift 2;; + -t|--to-registry) to_registry=$2; shift 2;; + -i|--image-name) image_name=$2; shift 2;; + --) break ;; + *) echo $1,$2,$show_usage; break ;; + esac + done + + d=`date` + echo "---$d: Merge $image_name from $from_registry to $to_registry" + + # Get new image tag + image_tag=`curl --cert /etc/docker-update-registry/update-registry.pem --key /etc/docker-update-registry/update-registry-key.pem --cacert /etc/docker-update-registry/ca.pem https://$from_registry/v2/$image_name/tags/list | awk -F'[' '{print $2}' | awk -F'"' '{print $2}'` + + echo "New image tag: $image_tag" + + # Check whether the image exists in the internal registry + ret_existing_image_tags=`curl --cert /etc/docker-registry/registry1.pem --key /etc/docker-registry/registry1-key.pem --cacert /etc/docker-registry/ca.pem https://$to_registry/v2/$image_name/tags/list` + + ret_key=`echo $ret_existing_image_tags | awk -F'"' '{print $2}'` + if [ $ret_key != "errors" ]; then + existing_image_tags=`echo $ret_existing_image_tags | awk -F'[' '{print $2}' | awk -F']' '{print $1}' | sed "s/\"//g"` + IFS=',' read -r -a existing_image_tag_list <<< "$existing_image_tags" + for existing_tag in ${existing_image_tag_list[@]}; do + if [ $existing_tag = $image_tag ]; then + echo "The image exists in registry. Skip." + exit 0 + fi + done + fi + + d=`date` + echo "---$d: Start to pull image" + docker pull $from_registry/$image_name:$image_tag + docker tag $from_registry/$image_name:$image_tag $to_registry/$image_name:$image_tag + + d=`date` + echo "---$d: Start to push image" + docker push $to_registry/$image_name:$image_tag + + d=`date` + echo "---$d: Start to clean local image" + docker rmi $from_registry/$image_name:$image_tag + docker rmi $to_registry/$image_name:$image_tag + + d=`date` + echo "---$d: End merging" +} + +IMGLIST=`sed "s/repositories:/""/g" <<< $3` +IMGLIST=`sed "s/{/""/g" <<< $IMGLIST` +IMGLIST=`sed "s/}/""/g" <<< $IMGLIST` +IMGLIST=`sed "s/\[/""/g" <<< $IMGLIST` +IMGLIST=`sed "s/\]/""/g" <<< $IMGLIST` +IMGLIST=`sed "s/\,/ /g" <<< $IMGLIST` + +for a in $IMGLIST; do conc "merge $1 $2 --image-name=$a";done;wait; diff --git a/utils/log/kubelet-plugins b/utils/log/kubelet-plugins new file mode 100644 index 0000000..38729f8 --- /dev/null +++ b/utils/log/kubelet-plugins @@ -0,0 +1,32 @@ +{{ kubelet_plugings_log_dir }}/*.log { + rotate 20 + maxage 30 + missingok + compress + } + +# TODO: clean up config when logrotate 3.10 is available https://superuser.com/questions/1059033/logrotate-wildcard-fails-on-missing-files-with-missingok-and-olddir +# TODO: check /var/lib/kubelet/plugins/**/*.log with newer version of logrotate +/var/lib/kubelet/plugins/*log /var/lib/kubelet/plugins/*/*log /var/lib/kubelet/plugins/*/*/*log /var/lib/kubelet/plugins/*/*/*/*log { + size 100M + missingok + copytruncate + sharedscripts + postrotate + rotated_logs="$(find /var/lib/kubelet/plugins -name '*.log.[1-20]')" + for file in $rotated_logs; do + log_file_name="$(basename $file | sed -r 's/.[0-9]+$//')" + mv $file {{ kubelet_plugings_log_dir }}/$log_file_name + done + endscript + } + +/var/log/plugin.log /var/log/danm.log { + size 15M + rotate 10 + maxage 30 + missingok + compress + copytruncate + dateext dateformat -%Y-%m-%d-%H-%s +} -- 2.16.6