Simplify ElasticSearch to make it more robust

- No more master/data nodes, only all-purpose ES nodes
- Move supervisord.conf to ES image from configmap

Signed-off-by: Ferenc Tóth <ferenc.2.toth@nokia.com>
Change-Id: Iaddc1f6aeb15e7188fb645ba72a955310d88383b

diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec
index 39588cb..11455d6 100644 (file)
--- a/SPECS/infra-charts.spec
+++ b/SPECS/infra-charts.spec
@@ -15,7 +15,7 @@
 %define COMPONENT infra-charts
 %define RPM_NAME caas-%{COMPONENT}
 %define RPM_MAJOR_VERSION 1.0.0
-%define RPM_MINOR_VERSION 34
+%define RPM_MINOR_VERSION 40
 
 Name:           %{RPM_NAME}
 Version:        %{RPM_MAJOR_VERSION}

diff --git a/cm_config/caas.yaml b/cm_config/caas.yaml
index b2a47ba..6d3ec73 100644 (file)
--- a/cm_config/caas.yaml
+++ b/cm_config/caas.yaml
@@ -53,16 +53,18 @@ registry_repo: "caas"
 update_registry_url: registry-update.kube-system.svc.{{ dns_domain }}
 update_registry_port: 5556
 update_registry: https://{{ update_registry_url }}:{{ update_registry_port }}
-elasticsearch_url: elasticsearch-logging.kube-system.svc.{{ dns_domain }}
+elasticsearch_url: elasticsearch.kube-system.svc.{{ dns_domain }}
 elasticsearch_port: 9200
 elasticsearch_scheme: http
 elasticsearch_logstash_prefix: caas
 elasticsearch_memory_request: 1Gi
 elasticsearch_memory_limit: 2Gi
+elasticsearch_cpu_request: 100m
+elasticsearch_cpu_limit: 1
 elasticsearch_java_heap_request: 800m
 elasticsearch_java_heap_max: 800m
 elasticsearch_port_for_cluster_communication: 9300
-elasticsearch: https://{{ elasticsearch_url }}:{{ elasticsearch_port }}
+elasticsearch: "{{ elasticsearch_scheme }}://{{ elasticsearch_url }}:{{ elasticsearch_port }}"
 internal_flannel_interface: eth0
 container_wait_timeout: 120
 ldap_master_url: master.infraldap-server.{{ dns_domain }}

diff --git a/infra-charts/templates/elasticsearch-configmap.yaml b/infra-charts/templates/elasticsearch-configmap.yaml
index b8ef093..56136c4 100644 (file)
--- a/infra-charts/templates/elasticsearch-configmap.yaml
+++ b/infra-charts/templates/elasticsearch-configmap.yaml
@@ -32,70 +32,10 @@ data:
     network.host: 0.0.0.0
     bootstrap.memory_lock: ${BOOTSTRAP_MEMORY_LOCK:false}
     discovery:
-      zen:
-        minimum_master_nodes: ${MINIMUM_MASTER_NODES}
-      seed_hosts: ${DISCOVERY_SERVICE}
-    processors: ${PROCESSORS:}
+      seed_hosts: ${DISCOVERY_SERVICE:}
     gateway.expected_master_nodes: ${EXPECTED_MASTER_NODES:3}
     gateway.expected_data_nodes: ${EXPECTED_DATA_NODES:3}
     gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m}
-    gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:3}
-    gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:3}
-
-  supervisord.conf: |
-    [unix_http_server]
-    file=/run/supervisord.sock   ; (the path to the socket file)
-    chmod = 0770
-    chown = root:root
-    username = dummy
-    password = dummy
-
-    [supervisord]
-    user=root
-    logfile=/dev/null
-    logfile_maxbytes=0
-    loglevel=info
-    pidfile=/var/run/supervisor/supervisord.pid
-    nodaemon=true
-    minfds=1024
-    minprocs=200
-
-    [supervisorctl]
-    serverurl=unix:///run/supervisord.sock
-    username = dummy
-    password = dummy
-
-    [rpcinterface:supervisor]
-    supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-    [program:elasticsearch]
-    command=/usr/share/elasticsearch/bin/elasticsearch
-    user= {{ .Values.elasticsearch.es_uid }}
-    startsecs=3
-    startretries=1
-    stopwaitsecs = 3
-    priority = 1
-    redirect_stderr=true
-    stdout_logfile=/dev/stdout
-    stdout_logfile_maxbytes=0
-
-    [program:crond]
-    command=/usr/sbin/crond -n -s
-    startsecs=3
-    startretries=1
-    stopwaitsecs = 3
-    priority = 2
-    redirect_stderr=true
-    stdout_logfile=/dev/stdout
-    stdout_logfile_maxbytes=0
-
-    [program:setindex]
-    command=/usr/share/elasticsearch/elasticsearch-setindex.sh
-    startsecs=3
-    startretries=1
-    stopwaitsecs = 3
-    priority = 2
-    redirect_stderr=true
-    stdout_logfile=/dev/stdout
-    stdout_logfile_maxbytes=0
+    gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:2}
+    gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:2}
 {{ end }}

diff --git a/infra-charts/templates/elasticsearch-data-statefulset.yaml b/infra-charts/templates/elasticsearch-data-statefulset.yaml
deleted file mode 100644 (file)
index 129d067..0000000
--- a/infra-charts/templates/elasticsearch-data-statefulset.yaml
+++ /dev/null
@@ -1,142 +0,0 @@
-{{/*
-Copyright 2019 Nokia
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-{{ if .Values.elasticsearch.required }}
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  labels:
-    app: elasticsearch-data
-    component: "{{ .Values.elasticsearch.data.name }}"
-  name: elasticsearch-data
-  namespace: kube-system
-spec:
-  serviceName: elasticsearch-data
-  replicas: {{ .Values.elasticsearch.data.replicas }}
-  selector:
-    matchLabels:
-      app: elasticsearch-data
-      component: "{{ .Values.elasticsearch.data.name }}"
-  template:
-    metadata:
-      labels:
-        app: elasticsearch-data
-        component: "{{ .Values.elasticsearch.data.name }}"
-    spec:
-      priorityClassName: "system-cluster-critical"
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - topologyKey: "kubernetes.io/hostname"
-              labelSelector:
-                matchLabels:
-                  app: elasticsearch-data
-                  component: "{{ .Values.elasticsearch.data.name }}"
-      nodeSelector:
-        nodetype: caas_master
-      containers:
-      - name: elasticsearch-data
-        env:
-        - name: node.name
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: metadata.name
-        - name: CLUSTER_INITIAL_MASTER_NODES
-          value: '{{ .Values.elasticsearch.clustermasternodes }}'
-        - name: EXPECTED_MASTER_NODES
-          value: '{{ .Values.elasticsearch.master.replicas }}'
-        - name: EXPECTED_DATA_NODES
-          value: '{{ .Values.elasticsearch.data.replicas }}'
-        - name: RECOVER_AFTER_MASTER_NODES
-          value: '{{ .Values.elasticsearch.master.replicas }}'
-        - name: RECOVER_AFTER_DATA_NODES
-          value: '{{ .Values.elasticsearch.data.replicas }}'
-        - name: NODE_MASTER
-          value: "false"
-        - name: NODE_INGEST
-          value: "true"
-        - name: NODE_DATA
-          value: "true"
-        - name: DISCOVERY_SERVICE
-          value: {{ .Values.elasticsearch.url }}
-        - name: PROCESSORS
-          valueFrom:
-            resourceFieldRef:
-              resource: limits.cpu
-        - name: ES_JAVA_OPTS
-          value: "-Xms{{ .Values.elasticsearch.java_heap_request }} -Xmx{{ .Values.elasticsearch.java_heap_max }} -Dmapper.allow_dots_in_name=true"
-        {{- range $key, $value :=  .Values.elasticsearch.env }}
-        - name: {{ $key }}
-          value: {{ $value | quote }}
-        {{- end }}
-        image: {{ .Values.elasticsearch.image_name }}
-        command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"]
-        resources:
-          limits:
-            cpu: "{{ .Values.elasticsearch.cpu_limit }}"
-            memory: "{{ .Values.elasticsearch.memory_limit }}"
-          requests:
-            cpu: "100m"
-            memory: "{{ .Values.elasticsearch.memory_request }}"
-        readinessProbe:
-          httpGet:
-            path: /_cluster/health?local=true
-            port: {{ .Values.elasticsearch.port }}
-          initialDelaySeconds: 5
-        livenessProbe:
-          httpGet:
-            path: /_cluster/health?local=true
-            port: {{ .Values.elasticsearch.port }}
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        volumeMounts:
-        - name: time-mount
-          mountPath: /etc/localtime
-          readOnly: true
-        - mountPath: /usr/share/elasticsearch/data
-          name: data
-        - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
-          name: elasticsearch-config
-          subPath: elasticsearch.yml
-        - mountPath: /etc/supervisord.conf
-          name: supervisord-config
-          subPath: supervisord.conf
-      terminationGracePeriodSeconds: {{ .Values.elasticsearch.data.terminationGracePeriodSeconds }}
-      volumes:
-        - name: time-mount
-          hostPath:
-            path: /etc/localtime
-        - name: elasticsearch-config
-          configMap:
-            name: elasticsearch
-            items:
-              - key: elasticsearch.yml
-                path: elasticsearch.yml
-                mode: 0644
-        - name: supervisord-config
-          configMap:
-            name: elasticsearch
-            items:
-              - key: supervisord.conf
-                path: supervisord.conf
-                mode: 0644
-        - name: data
-          hostPath:
-            path: /var/log/elasticsearch
-{{ end }}

diff --git a/infra-charts/templates/elasticsearch-service.yaml b/infra-charts/templates/elasticsearch-service.yaml
index 430cbcf..8a89e2a 100644 (file)
--- a/infra-charts/templates/elasticsearch-service.yaml
+++ b/infra-charts/templates/elasticsearch-service.yaml
@@ -19,9 +19,9 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    app: elasticsearch-data
-    component: "{{ .Values.elasticsearch.data.name }}"
-  name: elasticsearch-data
+    app: elasticsearch
+    component: "{{ .Values.elasticsearch.cluster.name }}"
+  name: elasticsearch
   namespace: kube-system
 spec:
   ports:
@@ -32,15 +32,15 @@ spec:
       name: node-to-node
       protocol: TCP
   selector:
-    app: elasticsearch-data
-    component: "{{ .Values.elasticsearch.data.name }}"
+    app: elasticsearch
+    component: "{{ .Values.elasticsearch.cluster.name }}"
 ---
 apiVersion: v1
 kind: Service
 metadata:
   labels:
-    app: elasticsearch-master
-    component: "{{ .Values.elasticsearch.master.name }}"
+    app: elasticsearch
+    component: "{{ .Values.elasticsearch.cluster.name }}"
   name: elasticsearch-logging
   namespace: kube-system
 spec:
@@ -48,10 +48,7 @@ spec:
     - port: {{ .Values.elasticsearch.port }}
       name: http
       protocol: TCP
-    - port: {{ .Values.elasticsearch.port_for_cluster_communication }}
-      name: node-to-node
-      protocol: TCP
   selector:
-    app: elasticsearch-master
-    component: "{{ .Values.elasticsearch.master.name }}"
+    app: elasticsearch
+    component: "{{ .Values.elasticsearch.cluster.name }}"
 {{ end }}

diff --git a/infra-charts/templates/elasticsearch-master-statefulset.yaml b/infra-charts/templates/elasticsearch-statefulset.yaml
similarity index 66%
rename from infra-charts/templates/elasticsearch-master-statefulset.yaml
rename to infra-charts/templates/elasticsearch-statefulset.yaml
index b31f272..608557b 100644 (file)
--- a/infra-charts/templates/elasticsearch-master-statefulset.yaml
+++ b/infra-charts/templates/elasticsearch-statefulset.yaml
@@ -19,22 +19,22 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   labels:
-    app: elasticsearch-master
-    component: "{{ .Values.elasticsearch.master.name }}"
-  name: elasticsearch-master
+    app: elasticsearch
+    component: "{{ .Values.elasticsearch.cluster.name }}"
+  name: elasticsearch
   namespace: kube-system
 spec:
-  serviceName: elasticsearch-master
-  replicas: {{ .Values.elasticsearch.master.replicas }}
+  serviceName: elasticsearch
+  replicas: {{ .Values.elasticsearch.replicas }}
   selector:
     matchLabels:
-      app: elasticsearch-master
-      component: "{{ .Values.elasticsearch.master.name }}"
+      app: elasticsearch
+      component: "{{ .Values.elasticsearch.cluster.name }}"
   template:
     metadata:
       labels:
-        app: elasticsearch-master
-        component: "{{ .Values.elasticsearch.master.name }}"
+        app: elasticsearch
+        component: "{{ .Values.elasticsearch.cluster.name }}"
     spec:
       priorityClassName: "system-cluster-critical"
       affinity:
@@ -43,46 +43,38 @@ spec:
             - topologyKey: "kubernetes.io/hostname"
               labelSelector:
                 matchLabels:
-                  app: elasticsearch-master
-                  component: "{{ .Values.elasticsearch.master.name }}"
+                  app: elasticsearch
+                  component: "{{ .Values.elasticsearch.cluster.name }}"
       nodeSelector:
         nodetype: caas_master
       containers:
-      - name: elasticsearch-master
+      - name: elasticsearch
         env:
         - name: node.name
           valueFrom:
             fieldRef:
+              apiVersion: v1
               fieldPath: metadata.name
-        - name: CLUSTER_INITIAL_MASTER_NODES
-          value: '{{ .Values.elasticsearch.clustermasternodes }}'
-        - name: EXPECTED_MASTER_NODES
-          value: '{{ .Values.elasticsearch.master.replicas }}'
-        - name: EXPECTED_DATA_NODES
-          value: '{{ .Values.elasticsearch.data.replicas }}'
-        - name: RECOVER_AFTER_MASTER_NODES
-          value: '{{ .Values.elasticsearch.master.replicas }}'
-        - name: RECOVER_AFTER_DATA_NODES
-          value: '{{ .Values.elasticsearch.data.replicas }}'
         - name: NODE_MASTER
           value: "true"
         - name: NODE_INGEST
-          value: "false"
+          value: "true"
         - name: NODE_DATA
-          value: "false"
-        - name: DISCOVERY_SERVICE
-          value: {{ .Values.elasticsearch.url }}
+          value: "true"
         - name: ES_JAVA_OPTS
           value: "-Xms{{ .Values.elasticsearch.java_heap_request }} -Xmx{{ .Values.elasticsearch.java_heap_max }} -Dmapper.allow_dots_in_name=true"
         {{- range $key, $value :=  .Values.elasticsearch.env }}
         - name: {{ $key }}
           value: {{ $value | quote }}
         {{- end }}
+        image: {{ .Values.elasticsearch.image_name }}
+        command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"]
         resources:
           limits:
+            cpu: "{{ .Values.elasticsearch.cpu_limit }}"
             memory: "{{ .Values.elasticsearch.memory_limit }}"
           requests:
-            cpu: "100m"
+            cpu: "{{ .Values.elasticsearch.cpu_request }}"
             memory: "{{ .Values.elasticsearch.memory_request }}"
         readinessProbe:
           httpGet:
@@ -97,8 +89,6 @@ spec:
           timeoutSeconds: 5
           successThreshold: 1
           failureThreshold: 5
-        image: {{ .Values.elasticsearch.image_name }}
-        command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"]
         volumeMounts:
         - name: time-mount
           mountPath: /etc/localtime
@@ -108,9 +98,8 @@ spec:
         - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
           name: elasticsearch-config
           subPath: elasticsearch.yml
-        - mountPath: /etc/supervisord.conf
-          name: supervisord-config
-          subPath: supervisord.conf
+          readOnly: true
+      terminationGracePeriodSeconds: {{ .Values.elasticsearch.terminationGracePeriodSeconds }}
       volumes:
         - name: time-mount
           hostPath:
@@ -122,13 +111,7 @@ spec:
               - key: elasticsearch.yml
                 path: elasticsearch.yml
                 mode: 0644
-        - name: supervisord-config
-          configMap:
-            name: elasticsearch
-            items:
-              - key: supervisord.conf
-                path: supervisord.conf
-                mode: 0644
         - name: data
-          emptyDir: {}
+          hostPath:
+            path: /var/log/elasticsearch
 {{ end }}

diff --git a/infra-charts/values.yaml.j2 b/infra-charts/values.yaml.j2
index 98c0752..b441ee6 100644 (file)
--- a/infra-charts/values.yaml.j2
+++ b/infra-charts/values.yaml.j2
@@ -76,38 +76,39 @@ danm_webhook:
   webhook_target_port: {{ caas.danm_webhook_target_port }}
 
 elasticsearch:
-{% set clustermasternodes = [] -%}
-{%- for masternumber in range(groups['caas_master']|length|int) -%}
-{%- if clustermasternodes.append('elasticsearch-master-' + (masternumber|string)) -%}{%- endif -%}
-{%- endfor %}
-  clustermasternodes: '{{ clustermasternodes|join(", ") }}'
 {% if caas.infra_log_store is defined and caas.infra_log_store != 'remote_syslog' %}
   required: true
 {% endif %}
+{%- set clustermasternodes = [] -%}
+{%- for masternumber in range(groups['caas_master']|length|int) -%}
+{%-   if clustermasternodes.append('elasticsearch-' + (masternumber|string)) -%}{%- endif -%}
+{%- endfor -%}
+{%- set cluster_master_nodes = clustermasternodes|join(",") %}
   url: {{ caas.elasticsearch_url }}
   port: {{ caas.elasticsearch_port }}
   scheme: {{ caas.elasticsearch_scheme }}
   logstash_prefix: {{ caas.elasticsearch_logstash_prefix }}
   image_name: {{ container_image_names | select('search', '/elasticsearch') | list | last }}
   memory_limit: {{ caas.elasticsearch_memory_limit }}
-  cpu_limit: 1
+  cpu_limit: {{ caas.elasticsearch_cpu_limit }}
   memory_request: {{ caas.elasticsearch_memory_request }}
+  cpu_request: {{ caas.elasticsearch_cpu_request }}
   java_heap_request: {{ caas.elasticsearch_java_heap_request }}
   java_heap_max: {{ caas.elasticsearch_java_heap_max }}
   port_for_cluster_communication: {{ caas.elasticsearch_port_for_cluster_communication }}
-  es_uid: {{ caas.uid.elasticsearch }}
   cluster:
     name: "elasticsearch"
+{% set minimum_master_node_num = (((groups['caas_master']|length|int)/2)|round(0, 'ceil'))|int %}
+{% set master_node_num = groups['caas_master']|length|int %}
   env:
-{% set minimummasternodes = ((((groups['caas_master']|length|int)/2)+1)|round(0, 'floor'))|int %}
-    MINIMUM_MASTER_NODES: {{ minimummasternodes }}
-  master:
-    name: elasticsearch-master
-    replicas: "{{ groups['caas_master']|length|int }}"
-  data:
-    name: elasticsearch-data
-    replicas: "{{ groups['caas_master']|length|int }}"
-    terminationGracePeriodSeconds: 3600
+    DISCOVERY_SERVICE: {{ caas.elasticsearch_url }}
+    CLUSTER_INITIAL_MASTER_NODES: {{ cluster_master_nodes }}
+    EXPECTED_MASTER_NODES: {{ master_node_num }}
+    EXPECTED_DATA_NODES: {{ master_node_num }}
+    RECOVER_AFTER_MASTER_NODES: {{ minimum_master_node_num }}
+    RECOVER_AFTER_MASTER_NODES: {{ minimum_master_node_num }}
+  replicas: {{ master_node_num }}
+  terminationGracePeriodSeconds: 3600
 
 sriovdp:
   required: true