Hybrid logging 12/1012/5
authorbborbely <botond.borbely@nokia.com>
Tue, 18 Jun 2019 12:10:17 +0000 (14:10 +0200)
committerbborbely <botond.borbely@nokia.com>
Wed, 26 Jun 2019 11:49:48 +0000 (13:49 +0200)
The fluentd component has moved to helm chart, too.
Now can be configured to use elasticsearch or remote_syslog for logging

Signed-off-by: bborbely <botond.borbely@nokia.com>
Change-Id: I795f48edf9213ede3c10ea249f76a12225ef3f5c

SPECS/infra-charts.spec
SPECS/instantiate.spec
ansible/roles/install_caas_infra/defaults/main.yaml
ansible/roles/install_caas_infra/tasks/main.yaml
cm_config/caas.yaml
infra-charts/templates/_helpers.tpl [new file with mode: 0644]
infra-charts/templates/elasticsearch-configmap.yaml
infra-charts/templates/fluentd-configmap.yaml [new file with mode: 0644]
infra-charts/templates/fluentd-daemonset.yaml [new file with mode: 0644]
infra-charts/values.yaml.j2
infra-charts/values_fluentd.yaml.j2 [new file with mode: 0644]

index bb2c1e2..54d3b2d 100644 (file)
@@ -15,7 +15,7 @@
 %define COMPONENT infra-charts
 %define RPM_NAME caas-%{COMPONENT}
 %define RPM_MAJOR_VERSION 1.0.0
-%define RPM_MINOR_VERSION 6
+%define RPM_MINOR_VERSION 7
 
 Name:           %{RPM_NAME}
 Version:        %{RPM_MAJOR_VERSION}
index ea2c0a0..b8cad86 100644 (file)
@@ -15,7 +15,7 @@
 %define COMPONENT instantiate
 %define RPM_NAME caas-%{COMPONENT}
 %define RPM_MAJOR_VERSION 1.0.0
-%define RPM_MINOR_VERSION 5
+%define RPM_MINOR_VERSION 6
 
 Name:           %{RPM_NAME}
 Version:        %{RPM_MAJOR_VERSION}
index c1b160d..83953cb 100644 (file)
@@ -22,3 +22,6 @@ infra_chart_images:
   - sriovdp
   - svcwatcher
   - netwatcher
+  - fluentd
+infra_chart_separated_values:
+  - fluentd
index a70fd64..b6b4ba5 100644 (file)
     src: "{{ caas.caas_chart_path }}/values.yaml.j2"
     dest: "{{ caas.caas_chart_path }}/values.yaml"
 
+- name: template component specific values file in helm dir
+  template:
+    src: "{{ caas.caas_chart_path }}/values_{{ item }}.yaml.j2"
+    dest: "{{ caas.caas_chart_path }}/values_{{ item }}.yaml"
+  with_items: "{{ infra_chart_separated_values }}"
+
+- name: append component specific values content to common values file
+  shell: "cat {{ caas.caas_chart_path }}/values_{{ item }}.yaml >> {{ caas.caas_chart_path }}/values.yaml"
+  with_items: "{{ infra_chart_separated_values }}"
+
 - set_fact:
     infra_chart_name: "caas-infra"
 
index cfb2fa6..327a9b9 100644 (file)
@@ -52,6 +52,8 @@ update_registry_port: 5556
 update_registry: https://{{ update_registry_url }}:{{ update_registry_port }}
 elasticsearch_url: elasticsearch-logging.kube-system.svc.{{ dns_domain }}
 elasticsearch_port: 9200
+elasticsearch_scheme: http
+elasticsearch_logstash_prefix: caas
 elasticsearch_memory_request: 1Gi
 elasticsearch_memory_limit: 2Gi
 elasticsearch_java_heap_request: 800m
@@ -101,6 +103,8 @@ uid:
   chartrepo: 150
   elasticsearch: 202
   flannel: 666
+infra_namespaces:
+  - kube-system
 cpu_allocation_mapping:
   caas_exclusive:
     pool_name: exclusive_caas
diff --git a/infra-charts/templates/_helpers.tpl b/infra-charts/templates/_helpers.tpl
new file mode 100644 (file)
index 0000000..e1e687e
--- /dev/null
@@ -0,0 +1,30 @@
+{{/*
+Copyright 2019 Nokia
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- define "caas.protocol_parser" }}
+{{- $url :=  regexSplit ":" . -1 }}
+       protocol {{ index $url 0 }}
+{{- end }}
+{{- define "caas.scheme_parser" }}
+{{- $url :=  regexSplit ":" . -1 }}
+       scheme {{ index $url 0 }}
+{{- end }}
+{{- define "caas.url_parser" }}
+{{- $url :=  regexSplit ":" . -1 }}
+{{- $just_url :=  index $url 1 }}
+{{- $just_url :=  regexSplit "\\/\\/" $just_url -1 }}
+       host {{ index $just_url 1 }}
+       port {{ index $url 2 }}
+{{- end }}
index 181c5dd..4e217d9 100644 (file)
@@ -137,7 +137,7 @@ data:
     do
         pcent=`df --output=pcent /usr/share/elasticsearch/data | tail -n1 | tr -d '%'`
         if [ $pcent -le $FS_LIMIT ]; then exit; fi
-        oldest=`curl -s -XGET 'localhost:9200/_cat/indices' | cut -d' ' -f3 | grep ^logstash | sort | head -n1`
-        curl -XDELETE "localhost:9200/$oldest" >/dev/null
+        oldest=`curl -s -XGET 'localhost:{{ .Values.elasticsearch.port }}/_cat/indices' | cut -d' ' -f3 | grep ^{{ .Values.elasticsearch.logstash_prefix }} | sort | head -n1`
+        curl -XDELETE "localhost:{{ .Values.elasticsearch.port }}/$oldest" >/dev/null
     done
 {{ end }}
diff --git a/infra-charts/templates/fluentd-configmap.yaml b/infra-charts/templates/fluentd-configmap.yaml
new file mode 100644 (file)
index 0000000..29894e3
--- /dev/null
@@ -0,0 +1,154 @@
+{{/*
+Copyright 2019 Nokia
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- define "es_infra.part" }}
+{{- template "es_begin.part"}}
+{{- with . }}
+       host {{ .host }}
+       port {{ .port }}
+       scheme {{ .scheme }}
+       logstash_prefix {{ .logstash_prefix }}
+{{- end }}
+       user elastic
+       password changeme
+{{- template "es_end.part"}}
+{{- end }}
+{{- define "es_log.part" }}
+{{- template "es_begin.part"}}
+{{- include "caas.url_parser" . }}
+{{- include "caas.scheme_parser" . }}
+       user elastic
+       password changeme
+{{- end }}
+{{- define "es_begin.part" }}
+    <match kubernetes**>
+       @type elasticsearch
+       @log_level info
+       include_tag_key true
+{{- end }}
+{{- define "es_end.part" }}
+       ssl_verify true
+       reload_connections true
+       logstash_format true
+       reconnect_on_error true
+       resurrect_after 5s
+       <buffer tag>
+          chunk_limit_size 256m
+          flush_at_shutdown true
+          flush_interval 5s
+          flush_thread_count 2
+       </buffer>
+    </match>
+{{- end }}
+{{- define "remote_syslog.part" }}
+    <match kubernetes**>
+      @type remote_syslog
+{{- if  . }}
+{{- include "caas.url_parser" . }}
+{{- include "caas.protocol_parser" . }}
+{{- else}}
+       host rsyslog-master
+       port 1025
+       protocol tcp
+{{- end }}
+      <buffer>
+        chunk_limit_size 256m
+        flush_at_shutdown "true"
+      </buffer>
+    </match>
+{{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-cm
+  namespace: kube-system
+data:
+  fluent.conf: |-
+    <system>
+{{- if .Values.fluentd.log_forwarding }}
+{{- $no_ws := len .Values.fluentd.log_forwarding }}
+      workers {{ add $no_ws 1 }}
+{{- else }}
+      workers 1
+{{- end }}
+      root_dir /fluentd
+    </system>
+    <match fluent**>
+      @type null
+    </match>
+    <worker 0>
+{{- range .Values.fluentd.infra_name_spaces }}
+      <source>
+        @type tail
+        path /var/log/containers/*_{{ . }}_*.log
+        pos_file /var/lib/fluentd/fluentd-{{ . }}.log.pos
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+        tag kubernetes.*
+        format json
+        read_from_head true
+      </source>
+{{- end }}
+      <filter kubernetes.**>
+        @type kubernetes_metadata
+        skip_labels true
+        skip_container_metadata true
+        skip_master_url true
+        skip_namespace_metadata true
+      </filter>
+{{- if .Values.elasticsearch.required}}
+{{ template "es_infra.part" .Values.elasticsearch }}
+{{- else }}
+{{ template "remote_syslog.part"  }}
+{{- end }}
+    </worker>
+{{- if .Values.fluentd.log_forwarding }}
+{{- range $index, $element := .Values.fluentd.log_forwarding }}
+    <worker {{ add $index 1 }}>
+      <source>
+        @type tail
+        path /var/log/containers/*_{{ $element.namespace }}-{{ $index }}_*.log
+        pos_file /var/lib/fluentd/fluentd-{{ $element.namespace }}-{{ $index }}.log.pos
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+        tag kubernetes.*
+        format json
+        read_from_head true
+      </source>
+      <filter kubernetes.**>
+        @type kubernetes_metadata
+        skip_labels true
+        skip_container_metadata true
+        skip_master_url true
+        skip_namespace_metadata true
+      </filter>
+{{- if ne $element.stream "both" }}
+      <filter kubernetes.**>
+        @type grep
+        <regexp>
+          key stream
+          pattern /{{ .stream }}/
+        </regexp>
+      </filter>
+{{- end }}
+{{- if eq $element.plugin "elasticsearch"}}
+{{ template "es_log.part" $element.target_url}}
+       logstash_prefix {{ $element.namespace }}
+{{- template "es_end.part"}}
+{{- else }}
+{{ template "remote_syslog.part" $element.target_url}}
+{{- end }}
+    </worker>
+{{- end }}
+{{- end }}
diff --git a/infra-charts/templates/fluentd-daemonset.yaml b/infra-charts/templates/fluentd-daemonset.yaml
new file mode 100644 (file)
index 0000000..79d0a18
--- /dev/null
@@ -0,0 +1,120 @@
+{{/*
+Copyright 2019 Nokia
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd
+  namespace: kube-system
+  labels:
+    {{ .Values.kubernetes_component_label }}: fluentd-logging
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    matchLabels:
+      {{ .Values.kubernetes_component_label }}: fluentd-logging
+  template:
+    metadata:
+      annotations:
+        config/timestamp: "{{ .Values.fluentd.config_timestamp }}"
+        danm.k8s.io/interfaces: |
+          [
+            {
+              "network":"flannel"
+            }
+          ]
+      labels:
+        {{ .Values.kubernetes_component_label }}: fluentd-logging
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      dnsPolicy: ClusterFirst
+      containers:
+      - name: fluentd
+        image: {{ .Values.fluentd.image_name }}
+        command:
+        - "/bin/entrypoint.sh"
+        - "fluentd"
+        - "-c"
+        - "/fluentd/etc/fluent.conf"
+        - "-p"
+        - "/fluentd/plugins"
+        securityContext:
+          capabilities:
+            add:
+              - ALL
+        env:
+          - name:  KUBERNETES_SERVICE_HOST
+            value: {{ .Values.fluentd.apiserver_service_name }}
+          - name:  KUBERNETES_SERVICE_PORT
+            value: "{{ .Values.fluentd.apiserver_svc_port }}"
+          - name:  FLUENT_ELASTICSEARCH_HOST
+            value: {{ .Values.elasticsearch.host }}
+          - name:  FLUENT_ELASTICSEARCH_PORT
+            value: "{{ .Values.elasticsearch.port }}"
+          - name: FLUENT_ELASTICSEARCH_SCHEME
+            value: {{ .Values.elasticsearch.scheme }}
+          - name: FLUENT_ELASTICSEARCH_USER
+            value: {{ .Values.elasticsearch.user | default "elastic" }}
+          - name: FLUENT_ELASTICSEARCH_PASSWORD
+            value: {{ .Values.elasticsearch.passwd  | default "changeme" }}
+          - name: FLUENT_ELASTICSEARCH_SSL
+            value: "false"
+          - name: FLUENTD_OPT
+            value: ""
+        resources:
+          requests:
+            cpu: 50m
+          limits:
+            cpu: 1
+        volumeMounts:
+        - name: time-mount
+          mountPath: /etc/localtime
+          readOnly: true
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+        - name: fluentd-secrets
+          mountPath: /etc/fluentd/ssl
+          readOnly: true
+        - name: fluentd-config
+          mountPath: /fluentd/etc
+          readOnly: true
+        - name: varlibfluentd
+          mountPath: /var/lib/fluentd
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: time-mount
+        hostPath:
+          path: /etc/localtime
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: fluentd-secrets
+        hostPath:
+          path: /etc/fluentd/ssl
+      - name: fluentd-config
+        configMap:
+          name: fluentd-cm
+      - name: varlibfluentd
+        hostPath:
+          path: /var/lib/fluentd
+      serviceAccountName: fluentd
index d87b6f6..6d34676 100644 (file)
@@ -65,9 +65,12 @@ elasticsearch:
 {%- if clustermasternodes.append('elasticsearch-master-' + (masternumber|string)) -%}{%- endif -%}
 {%- endfor %}
   clustermasternodes: '{{ clustermasternodes|join(", ") }}'
+{% if caas.infra_log_store is defined and caas.infra_log_store != 'remote_syslog' %}  
   required: true
-  url: {{ caas.elasticsearch_url }}
+{% endif %}
   port: {{ caas.elasticsearch_port }}
+  scheme: {{ caas.elasticsearch_scheme }}
+  logstash_prefix: {{ caas.elasticsearch_logstash_prefix }}
   image_name: {{ container_image_names | select('search', '/elasticsearch') | list | last }}
   memory_limit: {{ caas.elasticsearch_memory_limit }}
   cpu_limit: 1
diff --git a/infra-charts/values_fluentd.yaml.j2 b/infra-charts/values_fluentd.yaml.j2
new file mode 100644 (file)
index 0000000..736d8b5
--- /dev/null
@@ -0,0 +1,42 @@
+{#
+Copyright 2019 Nokia
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+#}
+---
+fluentd:
+  image_name: {{ container_image_names | select('search', '/fluentd') | list | last }}
+  apiserver_service_name: {{ caas.apiserver_service_name }}
+  apiserver_svc_port: {{ caas.apiserver_svc_port }}
+  infra_name_spaces:
+{% if caas.infra_namespaces is defined %}
+{%- for item in caas.infra_namespaces %}
+  - {{ item }}
+{% endfor %}
+{% else %}
+  - kube-system
+{%- endif %}
+{%- if caas.log_forwarding is defined %}
+  log_forwarding:
+{% for item in caas.log_forwarding %}
+  - namespace: {{ item.namespace }}
+    target_url: "{{ item.target_url }}"
+    plugin: {{ item.plugin}}
+{% if item.stream is defined %}
+    stream: {{ item.stream}}
+{% else %}
+    stream: both
+{% endif %}
+{%- endfor %}
+{% endif %}
+  config_timestamp: {{ ansible_date_time.epoch }}