From 3160ed6efbb49efe9a855a25782a20335da1eef5 Mon Sep 17 00:00:00 2001 From: bborbely Date: Tue, 18 Jun 2019 14:10:17 +0200 Subject: [PATCH] Hybrid logging The fluentd component has moved to helm chart, too. Now can be configured to use elasticsearch or remote_syslog for logging Signed-off-by: bborbely Change-Id: I795f48edf9213ede3c10ea249f76a12225ef3f5c --- SPECS/infra-charts.spec | 2 +- SPECS/instantiate.spec | 2 +- .../roles/install_caas_infra/defaults/main.yaml | 3 + ansible/roles/install_caas_infra/tasks/main.yaml | 10 ++ cm_config/caas.yaml | 4 + infra-charts/templates/_helpers.tpl | 30 ++++ .../templates/elasticsearch-configmap.yaml | 4 +- infra-charts/templates/fluentd-configmap.yaml | 154 +++++++++++++++++++++ infra-charts/templates/fluentd-daemonset.yaml | 120 ++++++++++++++++ infra-charts/values.yaml.j2 | 5 +- infra-charts/values_fluentd.yaml.j2 | 42 ++++++ 11 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 infra-charts/templates/_helpers.tpl create mode 100644 infra-charts/templates/fluentd-configmap.yaml create mode 100644 infra-charts/templates/fluentd-daemonset.yaml create mode 100644 infra-charts/values_fluentd.yaml.j2 diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec index bb2c1e2..54d3b2d 100644 --- a/SPECS/infra-charts.spec +++ b/SPECS/infra-charts.spec @@ -15,7 +15,7 @@ %define COMPONENT infra-charts %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 1.0.0 -%define RPM_MINOR_VERSION 6 +%define RPM_MINOR_VERSION 7 Name: %{RPM_NAME} Version: %{RPM_MAJOR_VERSION} diff --git a/SPECS/instantiate.spec b/SPECS/instantiate.spec index ea2c0a0..b8cad86 100644 --- a/SPECS/instantiate.spec +++ b/SPECS/instantiate.spec @@ -15,7 +15,7 @@ %define COMPONENT instantiate %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 1.0.0 -%define RPM_MINOR_VERSION 5 +%define RPM_MINOR_VERSION 6 Name: %{RPM_NAME} Version: %{RPM_MAJOR_VERSION} diff --git a/ansible/roles/install_caas_infra/defaults/main.yaml b/ansible/roles/install_caas_infra/defaults/main.yaml index c1b160d..83953cb 100644 --- a/ansible/roles/install_caas_infra/defaults/main.yaml +++ b/ansible/roles/install_caas_infra/defaults/main.yaml @@ -22,3 +22,6 @@ infra_chart_images: - sriovdp - svcwatcher - netwatcher + - fluentd +infra_chart_separated_values: + - fluentd diff --git a/ansible/roles/install_caas_infra/tasks/main.yaml b/ansible/roles/install_caas_infra/tasks/main.yaml index a70fd64..b6b4ba5 100644 --- a/ansible/roles/install_caas_infra/tasks/main.yaml +++ b/ansible/roles/install_caas_infra/tasks/main.yaml @@ -18,6 +18,16 @@ src: "{{ caas.caas_chart_path }}/values.yaml.j2" dest: "{{ caas.caas_chart_path }}/values.yaml" +- name: template component specific values file in helm dir + template: + src: "{{ caas.caas_chart_path }}/values_{{ item }}.yaml.j2" + dest: "{{ caas.caas_chart_path }}/values_{{ item }}.yaml" + with_items: "{{ infra_chart_separated_values }}" + +- name: append component specific values content to common values file + shell: "cat {{ caas.caas_chart_path }}/values_{{ item }}.yaml >> {{ caas.caas_chart_path }}/values.yaml" + with_items: "{{ infra_chart_separated_values }}" + - set_fact: infra_chart_name: "caas-infra" diff --git a/cm_config/caas.yaml b/cm_config/caas.yaml index cfb2fa6..327a9b9 100644 --- a/cm_config/caas.yaml +++ b/cm_config/caas.yaml @@ -52,6 +52,8 @@ update_registry_port: 5556 update_registry: https://{{ update_registry_url }}:{{ update_registry_port }} elasticsearch_url: elasticsearch-logging.kube-system.svc.{{ dns_domain }} elasticsearch_port: 9200 +elasticsearch_scheme: http +elasticsearch_logstash_prefix: caas elasticsearch_memory_request: 1Gi elasticsearch_memory_limit: 2Gi elasticsearch_java_heap_request: 800m @@ -101,6 +103,8 @@ uid: chartrepo: 150 elasticsearch: 202 flannel: 666 +infra_namespaces: + - kube-system cpu_allocation_mapping: caas_exclusive: pool_name: exclusive_caas diff --git a/infra-charts/templates/_helpers.tpl b/infra-charts/templates/_helpers.tpl new file mode 100644 index 0000000..e1e687e --- /dev/null +++ b/infra-charts/templates/_helpers.tpl @@ -0,0 +1,30 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "caas.protocol_parser" }} +{{- $url := regexSplit ":" . -1 }} + protocol {{ index $url 0 }} +{{- end }} +{{- define "caas.scheme_parser" }} +{{- $url := regexSplit ":" . -1 }} + scheme {{ index $url 0 }} +{{- end }} +{{- define "caas.url_parser" }} +{{- $url := regexSplit ":" . -1 }} +{{- $just_url := index $url 1 }} +{{- $just_url := regexSplit "\\/\\/" $just_url -1 }} + host {{ index $just_url 1 }} + port {{ index $url 2 }} +{{- end }} diff --git a/infra-charts/templates/elasticsearch-configmap.yaml b/infra-charts/templates/elasticsearch-configmap.yaml index 181c5dd..4e217d9 100644 --- a/infra-charts/templates/elasticsearch-configmap.yaml +++ b/infra-charts/templates/elasticsearch-configmap.yaml @@ -137,7 +137,7 @@ data: do pcent=`df --output=pcent /usr/share/elasticsearch/data | tail -n1 | tr -d '%'` if [ $pcent -le $FS_LIMIT ]; then exit; fi - oldest=`curl -s -XGET 'localhost:9200/_cat/indices' | cut -d' ' -f3 | grep ^logstash | sort | head -n1` - curl -XDELETE "localhost:9200/$oldest" >/dev/null + oldest=`curl -s -XGET 'localhost:{{ .Values.elasticsearch.port }}/_cat/indices' | cut -d' ' -f3 | grep ^{{ .Values.elasticsearch.logstash_prefix }} | sort | head -n1` + curl -XDELETE "localhost:{{ .Values.elasticsearch.port }}/$oldest" >/dev/null done {{ end }} diff --git a/infra-charts/templates/fluentd-configmap.yaml b/infra-charts/templates/fluentd-configmap.yaml new file mode 100644 index 0000000..29894e3 --- /dev/null +++ b/infra-charts/templates/fluentd-configmap.yaml @@ -0,0 +1,154 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "es_infra.part" }} +{{- template "es_begin.part"}} +{{- with . }} + host {{ .host }} + port {{ .port }} + scheme {{ .scheme }} + logstash_prefix {{ .logstash_prefix }} +{{- end }} + user elastic + password changeme +{{- template "es_end.part"}} +{{- end }} +{{- define "es_log.part" }} +{{- template "es_begin.part"}} +{{- include "caas.url_parser" . }} +{{- include "caas.scheme_parser" . }} + user elastic + password changeme +{{- end }} +{{- define "es_begin.part" }} + + @type elasticsearch + @log_level info + include_tag_key true +{{- end }} +{{- define "es_end.part" }} + ssl_verify true + reload_connections true + logstash_format true + reconnect_on_error true + resurrect_after 5s + + chunk_limit_size 256m + flush_at_shutdown true + flush_interval 5s + flush_thread_count 2 + + +{{- end }} +{{- define "remote_syslog.part" }} + + @type remote_syslog +{{- if . }} +{{- include "caas.url_parser" . }} +{{- include "caas.protocol_parser" . }} +{{- else}} + host rsyslog-master + port 1025 + protocol tcp +{{- end }} + + chunk_limit_size 256m + flush_at_shutdown "true" + + +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-cm + namespace: kube-system +data: + fluent.conf: |- + +{{- if .Values.fluentd.log_forwarding }} +{{- $no_ws := len .Values.fluentd.log_forwarding }} + workers {{ add $no_ws 1 }} +{{- else }} + workers 1 +{{- end }} + root_dir /fluentd + + + @type null + + +{{- range .Values.fluentd.infra_name_spaces }} + + @type tail + path /var/log/containers/*_{{ . }}_*.log + pos_file /var/lib/fluentd/fluentd-{{ . }}.log.pos + time_format %Y-%m-%dT%H:%M:%S.%NZ + tag kubernetes.* + format json + read_from_head true + +{{- end }} + + @type kubernetes_metadata + skip_labels true + skip_container_metadata true + skip_master_url true + skip_namespace_metadata true + +{{- if .Values.elasticsearch.required}} +{{ template "es_infra.part" .Values.elasticsearch }} +{{- else }} +{{ template "remote_syslog.part" }} +{{- end }} + +{{- if .Values.fluentd.log_forwarding }} +{{- range $index, $element := .Values.fluentd.log_forwarding }} + + + @type tail + path /var/log/containers/*_{{ $element.namespace }}-{{ $index }}_*.log + pos_file /var/lib/fluentd/fluentd-{{ $element.namespace }}-{{ $index }}.log.pos + time_format %Y-%m-%dT%H:%M:%S.%NZ + tag kubernetes.* + format json + read_from_head true + + + @type kubernetes_metadata + skip_labels true + skip_container_metadata true + skip_master_url true + skip_namespace_metadata true + +{{- if ne $element.stream "both" }} + + @type grep + + key stream + pattern /{{ .stream }}/ + + +{{- end }} +{{- if eq $element.plugin "elasticsearch"}} +{{ template "es_log.part" $element.target_url}} + logstash_prefix {{ $element.namespace }} +{{- template "es_end.part"}} +{{- else }} +{{ template "remote_syslog.part" $element.target_url}} +{{- end }} + +{{- end }} +{{- end }} diff --git a/infra-charts/templates/fluentd-daemonset.yaml b/infra-charts/templates/fluentd-daemonset.yaml new file mode 100644 index 0000000..79d0a18 --- /dev/null +++ b/infra-charts/templates/fluentd-daemonset.yaml @@ -0,0 +1,120 @@ +{{/* +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: kube-system + labels: + {{ .Values.kubernetes_component_label }}: fluentd-logging + version: v1 + kubernetes.io/cluster-service: "true" +spec: + selector: + matchLabels: + {{ .Values.kubernetes_component_label }}: fluentd-logging + template: + metadata: + annotations: + config/timestamp: "{{ .Values.fluentd.config_timestamp }}" + danm.k8s.io/interfaces: | + [ + { + "network":"flannel" + } + ] + labels: + {{ .Values.kubernetes_component_label }}: fluentd-logging + version: v1 + kubernetes.io/cluster-service: "true" + spec: + dnsPolicy: ClusterFirst + containers: + - name: fluentd + image: {{ .Values.fluentd.image_name }} + command: + - "/bin/entrypoint.sh" + - "fluentd" + - "-c" + - "/fluentd/etc/fluent.conf" + - "-p" + - "/fluentd/plugins" + securityContext: + capabilities: + add: + - ALL + env: + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.fluentd.apiserver_service_name }} + - name: KUBERNETES_SERVICE_PORT + value: "{{ .Values.fluentd.apiserver_svc_port }}" + - name: FLUENT_ELASTICSEARCH_HOST + value: {{ .Values.elasticsearch.host }} + - name: FLUENT_ELASTICSEARCH_PORT + value: "{{ .Values.elasticsearch.port }}" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: {{ .Values.elasticsearch.scheme }} + - name: FLUENT_ELASTICSEARCH_USER + value: {{ .Values.elasticsearch.user | default "elastic" }} + - name: FLUENT_ELASTICSEARCH_PASSWORD + value: {{ .Values.elasticsearch.passwd | default "changeme" }} + - name: FLUENT_ELASTICSEARCH_SSL + value: "false" + - name: FLUENTD_OPT + value: "" + resources: + requests: + cpu: 50m + limits: + cpu: 1 + volumeMounts: + - name: time-mount + mountPath: /etc/localtime + readOnly: true + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + - name: fluentd-secrets + mountPath: /etc/fluentd/ssl + readOnly: true + - name: fluentd-config + mountPath: /fluentd/etc + readOnly: true + - name: varlibfluentd + mountPath: /var/lib/fluentd + terminationGracePeriodSeconds: 30 + volumes: + - name: time-mount + hostPath: + path: /etc/localtime + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: fluentd-secrets + hostPath: + path: /etc/fluentd/ssl + - name: fluentd-config + configMap: + name: fluentd-cm + - name: varlibfluentd + hostPath: + path: /var/lib/fluentd + serviceAccountName: fluentd diff --git a/infra-charts/values.yaml.j2 b/infra-charts/values.yaml.j2 index d87b6f6..6d34676 100644 --- a/infra-charts/values.yaml.j2 +++ b/infra-charts/values.yaml.j2 @@ -65,9 +65,12 @@ elasticsearch: {%- if clustermasternodes.append('elasticsearch-master-' + (masternumber|string)) -%}{%- endif -%} {%- endfor %} clustermasternodes: '{{ clustermasternodes|join(", ") }}' +{% if caas.infra_log_store is defined and caas.infra_log_store != 'remote_syslog' %} required: true - url: {{ caas.elasticsearch_url }} +{% endif %} port: {{ caas.elasticsearch_port }} + scheme: {{ caas.elasticsearch_scheme }} + logstash_prefix: {{ caas.elasticsearch_logstash_prefix }} image_name: {{ container_image_names | select('search', '/elasticsearch') | list | last }} memory_limit: {{ caas.elasticsearch_memory_limit }} cpu_limit: 1 diff --git a/infra-charts/values_fluentd.yaml.j2 b/infra-charts/values_fluentd.yaml.j2 new file mode 100644 index 0000000..736d8b5 --- /dev/null +++ b/infra-charts/values_fluentd.yaml.j2 @@ -0,0 +1,42 @@ +{# +Copyright 2019 Nokia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +--- +fluentd: + image_name: {{ container_image_names | select('search', '/fluentd') | list | last }} + apiserver_service_name: {{ caas.apiserver_service_name }} + apiserver_svc_port: {{ caas.apiserver_svc_port }} + infra_name_spaces: +{% if caas.infra_namespaces is defined %} +{%- for item in caas.infra_namespaces %} + - {{ item }} +{% endfor %} +{% else %} + - kube-system +{%- endif %} +{%- if caas.log_forwarding is defined %} + log_forwarding: +{% for item in caas.log_forwarding %} + - namespace: {{ item.namespace }} + target_url: "{{ item.target_url }}" + plugin: {{ item.plugin}} +{% if item.stream is defined %} + stream: {{ item.stream}} +{% else %} + stream: both +{% endif %} +{%- endfor %} +{% endif %} + config_timestamp: {{ ansible_date_time.epoch }} -- 2.16.6