From bec9881f04d6957dc0f828af2c8f9247c8b9f568 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ferenc=20T=C3=B3th?= Date: Mon, 14 Oct 2019 22:11:57 +0200 Subject: [PATCH] Simplify ElasticSearch to make it more robust MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit - No more master/data nodes, only all-purpose ES nodes - Move supervisord.conf to ES image from configmap Signed-off-by: Ferenc Tóth Change-Id: Iaddc1f6aeb15e7188fb645ba72a955310d88383b --- SPECS/infra-charts.spec | 2 +- cm_config/caas.yaml | 6 +- .../templates/elasticsearch-configmap.yaml | 66 +--------- .../templates/elasticsearch-data-statefulset.yaml | 142 --------------------- infra-charts/templates/elasticsearch-service.yaml | 21 ++- ...efulset.yaml => elasticsearch-statefulset.yaml} | 63 ++++----- infra-charts/values.yaml.j2 | 33 ++--- 7 files changed, 57 insertions(+), 276 deletions(-) delete mode 100644 infra-charts/templates/elasticsearch-data-statefulset.yaml rename infra-charts/templates/{elasticsearch-master-statefulset.yaml => elasticsearch-statefulset.yaml} (66%) diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec index 39588cb..11455d6 100644 --- a/SPECS/infra-charts.spec +++ b/SPECS/infra-charts.spec @@ -15,7 +15,7 @@ %define COMPONENT infra-charts %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 1.0.0 -%define RPM_MINOR_VERSION 34 +%define RPM_MINOR_VERSION 40 Name: %{RPM_NAME} Version: %{RPM_MAJOR_VERSION} diff --git a/cm_config/caas.yaml b/cm_config/caas.yaml index b2a47ba..6d3ec73 100644 --- a/cm_config/caas.yaml +++ b/cm_config/caas.yaml @@ -53,16 +53,18 @@ registry_repo: "caas" update_registry_url: registry-update.kube-system.svc.{{ dns_domain }} update_registry_port: 5556 update_registry: https://{{ update_registry_url }}:{{ update_registry_port }} -elasticsearch_url: elasticsearch-logging.kube-system.svc.{{ dns_domain }} +elasticsearch_url: elasticsearch.kube-system.svc.{{ dns_domain }} elasticsearch_port: 9200 elasticsearch_scheme: http elasticsearch_logstash_prefix: caas elasticsearch_memory_request: 1Gi elasticsearch_memory_limit: 2Gi +elasticsearch_cpu_request: 100m +elasticsearch_cpu_limit: 1 elasticsearch_java_heap_request: 800m elasticsearch_java_heap_max: 800m elasticsearch_port_for_cluster_communication: 9300 -elasticsearch: https://{{ elasticsearch_url }}:{{ elasticsearch_port }} +elasticsearch: "{{ elasticsearch_scheme }}://{{ elasticsearch_url }}:{{ elasticsearch_port }}" internal_flannel_interface: eth0 container_wait_timeout: 120 ldap_master_url: master.infraldap-server.{{ dns_domain }} diff --git a/infra-charts/templates/elasticsearch-configmap.yaml b/infra-charts/templates/elasticsearch-configmap.yaml index b8ef093..56136c4 100644 --- a/infra-charts/templates/elasticsearch-configmap.yaml +++ b/infra-charts/templates/elasticsearch-configmap.yaml @@ -32,70 +32,10 @@ data: network.host: 0.0.0.0 bootstrap.memory_lock: ${BOOTSTRAP_MEMORY_LOCK:false} discovery: - zen: - minimum_master_nodes: ${MINIMUM_MASTER_NODES} - seed_hosts: ${DISCOVERY_SERVICE} - processors: ${PROCESSORS:} + seed_hosts: ${DISCOVERY_SERVICE:} gateway.expected_master_nodes: ${EXPECTED_MASTER_NODES:3} gateway.expected_data_nodes: ${EXPECTED_DATA_NODES:3} gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m} - gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:3} - gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:3} - - supervisord.conf: | - [unix_http_server] - file=/run/supervisord.sock ; (the path to the socket file) - chmod = 0770 - chown = root:root - username = dummy - password = dummy - - [supervisord] - user=root - logfile=/dev/null - logfile_maxbytes=0 - loglevel=info - pidfile=/var/run/supervisor/supervisord.pid - nodaemon=true - minfds=1024 - minprocs=200 - - [supervisorctl] - serverurl=unix:///run/supervisord.sock - username = dummy - password = dummy - - [rpcinterface:supervisor] - supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - - [program:elasticsearch] - command=/usr/share/elasticsearch/bin/elasticsearch - user= {{ .Values.elasticsearch.es_uid }} - startsecs=3 - startretries=1 - stopwaitsecs = 3 - priority = 1 - redirect_stderr=true - stdout_logfile=/dev/stdout - stdout_logfile_maxbytes=0 - - [program:crond] - command=/usr/sbin/crond -n -s - startsecs=3 - startretries=1 - stopwaitsecs = 3 - priority = 2 - redirect_stderr=true - stdout_logfile=/dev/stdout - stdout_logfile_maxbytes=0 - - [program:setindex] - command=/usr/share/elasticsearch/elasticsearch-setindex.sh - startsecs=3 - startretries=1 - stopwaitsecs = 3 - priority = 2 - redirect_stderr=true - stdout_logfile=/dev/stdout - stdout_logfile_maxbytes=0 + gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:2} + gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:2} {{ end }} diff --git a/infra-charts/templates/elasticsearch-data-statefulset.yaml b/infra-charts/templates/elasticsearch-data-statefulset.yaml deleted file mode 100644 index 129d067..0000000 --- a/infra-charts/templates/elasticsearch-data-statefulset.yaml +++ /dev/null @@ -1,142 +0,0 @@ -{{/* -Copyright 2019 Nokia - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} -{{ if .Values.elasticsearch.required }} ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - app: elasticsearch-data - component: "{{ .Values.elasticsearch.data.name }}" - name: elasticsearch-data - namespace: kube-system -spec: - serviceName: elasticsearch-data - replicas: {{ .Values.elasticsearch.data.replicas }} - selector: - matchLabels: - app: elasticsearch-data - component: "{{ .Values.elasticsearch.data.name }}" - template: - metadata: - labels: - app: elasticsearch-data - component: "{{ .Values.elasticsearch.data.name }}" - spec: - priorityClassName: "system-cluster-critical" - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: "kubernetes.io/hostname" - labelSelector: - matchLabels: - app: elasticsearch-data - component: "{{ .Values.elasticsearch.data.name }}" - nodeSelector: - nodetype: caas_master - containers: - - name: elasticsearch-data - env: - - name: node.name - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: CLUSTER_INITIAL_MASTER_NODES - value: '{{ .Values.elasticsearch.clustermasternodes }}' - - name: EXPECTED_MASTER_NODES - value: '{{ .Values.elasticsearch.master.replicas }}' - - name: EXPECTED_DATA_NODES - value: '{{ .Values.elasticsearch.data.replicas }}' - - name: RECOVER_AFTER_MASTER_NODES - value: '{{ .Values.elasticsearch.master.replicas }}' - - name: RECOVER_AFTER_DATA_NODES - value: '{{ .Values.elasticsearch.data.replicas }}' - - name: NODE_MASTER - value: "false" - - name: NODE_INGEST - value: "true" - - name: NODE_DATA - value: "true" - - name: DISCOVERY_SERVICE - value: {{ .Values.elasticsearch.url }} - - name: PROCESSORS - valueFrom: - resourceFieldRef: - resource: limits.cpu - - name: ES_JAVA_OPTS - value: "-Xms{{ .Values.elasticsearch.java_heap_request }} -Xmx{{ .Values.elasticsearch.java_heap_max }} -Dmapper.allow_dots_in_name=true" - {{- range $key, $value := .Values.elasticsearch.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - image: {{ .Values.elasticsearch.image_name }} - command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"] - resources: - limits: - cpu: "{{ .Values.elasticsearch.cpu_limit }}" - memory: "{{ .Values.elasticsearch.memory_limit }}" - requests: - cpu: "100m" - memory: "{{ .Values.elasticsearch.memory_request }}" - readinessProbe: - httpGet: - path: /_cluster/health?local=true - port: {{ .Values.elasticsearch.port }} - initialDelaySeconds: 5 - livenessProbe: - httpGet: - path: /_cluster/health?local=true - port: {{ .Values.elasticsearch.port }} - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - volumeMounts: - - name: time-mount - mountPath: /etc/localtime - readOnly: true - - mountPath: /usr/share/elasticsearch/data - name: data - - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml - name: elasticsearch-config - subPath: elasticsearch.yml - - mountPath: /etc/supervisord.conf - name: supervisord-config - subPath: supervisord.conf - terminationGracePeriodSeconds: {{ .Values.elasticsearch.data.terminationGracePeriodSeconds }} - volumes: - - name: time-mount - hostPath: - path: /etc/localtime - - name: elasticsearch-config - configMap: - name: elasticsearch - items: - - key: elasticsearch.yml - path: elasticsearch.yml - mode: 0644 - - name: supervisord-config - configMap: - name: elasticsearch - items: - - key: supervisord.conf - path: supervisord.conf - mode: 0644 - - name: data - hostPath: - path: /var/log/elasticsearch -{{ end }} diff --git a/infra-charts/templates/elasticsearch-service.yaml b/infra-charts/templates/elasticsearch-service.yaml index 430cbcf..8a89e2a 100644 --- a/infra-charts/templates/elasticsearch-service.yaml +++ b/infra-charts/templates/elasticsearch-service.yaml @@ -19,9 +19,9 @@ apiVersion: v1 kind: Service metadata: labels: - app: elasticsearch-data - component: "{{ .Values.elasticsearch.data.name }}" - name: elasticsearch-data + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" + name: elasticsearch namespace: kube-system spec: ports: @@ -32,15 +32,15 @@ spec: name: node-to-node protocol: TCP selector: - app: elasticsearch-data - component: "{{ .Values.elasticsearch.data.name }}" + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" --- apiVersion: v1 kind: Service metadata: labels: - app: elasticsearch-master - component: "{{ .Values.elasticsearch.master.name }}" + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" name: elasticsearch-logging namespace: kube-system spec: @@ -48,10 +48,7 @@ spec: - port: {{ .Values.elasticsearch.port }} name: http protocol: TCP - - port: {{ .Values.elasticsearch.port_for_cluster_communication }} - name: node-to-node - protocol: TCP selector: - app: elasticsearch-master - component: "{{ .Values.elasticsearch.master.name }}" + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" {{ end }} diff --git a/infra-charts/templates/elasticsearch-master-statefulset.yaml b/infra-charts/templates/elasticsearch-statefulset.yaml similarity index 66% rename from infra-charts/templates/elasticsearch-master-statefulset.yaml rename to infra-charts/templates/elasticsearch-statefulset.yaml index b31f272..608557b 100644 --- a/infra-charts/templates/elasticsearch-master-statefulset.yaml +++ b/infra-charts/templates/elasticsearch-statefulset.yaml @@ -19,22 +19,22 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: - app: elasticsearch-master - component: "{{ .Values.elasticsearch.master.name }}" - name: elasticsearch-master + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" + name: elasticsearch namespace: kube-system spec: - serviceName: elasticsearch-master - replicas: {{ .Values.elasticsearch.master.replicas }} + serviceName: elasticsearch + replicas: {{ .Values.elasticsearch.replicas }} selector: matchLabels: - app: elasticsearch-master - component: "{{ .Values.elasticsearch.master.name }}" + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" template: metadata: labels: - app: elasticsearch-master - component: "{{ .Values.elasticsearch.master.name }}" + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" spec: priorityClassName: "system-cluster-critical" affinity: @@ -43,46 +43,38 @@ spec: - topologyKey: "kubernetes.io/hostname" labelSelector: matchLabels: - app: elasticsearch-master - component: "{{ .Values.elasticsearch.master.name }}" + app: elasticsearch + component: "{{ .Values.elasticsearch.cluster.name }}" nodeSelector: nodetype: caas_master containers: - - name: elasticsearch-master + - name: elasticsearch env: - name: node.name valueFrom: fieldRef: + apiVersion: v1 fieldPath: metadata.name - - name: CLUSTER_INITIAL_MASTER_NODES - value: '{{ .Values.elasticsearch.clustermasternodes }}' - - name: EXPECTED_MASTER_NODES - value: '{{ .Values.elasticsearch.master.replicas }}' - - name: EXPECTED_DATA_NODES - value: '{{ .Values.elasticsearch.data.replicas }}' - - name: RECOVER_AFTER_MASTER_NODES - value: '{{ .Values.elasticsearch.master.replicas }}' - - name: RECOVER_AFTER_DATA_NODES - value: '{{ .Values.elasticsearch.data.replicas }}' - name: NODE_MASTER value: "true" - name: NODE_INGEST - value: "false" + value: "true" - name: NODE_DATA - value: "false" - - name: DISCOVERY_SERVICE - value: {{ .Values.elasticsearch.url }} + value: "true" - name: ES_JAVA_OPTS value: "-Xms{{ .Values.elasticsearch.java_heap_request }} -Xmx{{ .Values.elasticsearch.java_heap_max }} -Dmapper.allow_dots_in_name=true" {{- range $key, $value := .Values.elasticsearch.env }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} + image: {{ .Values.elasticsearch.image_name }} + command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"] resources: limits: + cpu: "{{ .Values.elasticsearch.cpu_limit }}" memory: "{{ .Values.elasticsearch.memory_limit }}" requests: - cpu: "100m" + cpu: "{{ .Values.elasticsearch.cpu_request }}" memory: "{{ .Values.elasticsearch.memory_request }}" readinessProbe: httpGet: @@ -97,8 +89,6 @@ spec: timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 - image: {{ .Values.elasticsearch.image_name }} - command: ["/usr/bin/supervisord","-n","-c","/etc/supervisord.conf"] volumeMounts: - name: time-mount mountPath: /etc/localtime @@ -108,9 +98,8 @@ spec: - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml name: elasticsearch-config subPath: elasticsearch.yml - - mountPath: /etc/supervisord.conf - name: supervisord-config - subPath: supervisord.conf + readOnly: true + terminationGracePeriodSeconds: {{ .Values.elasticsearch.terminationGracePeriodSeconds }} volumes: - name: time-mount hostPath: @@ -122,13 +111,7 @@ spec: - key: elasticsearch.yml path: elasticsearch.yml mode: 0644 - - name: supervisord-config - configMap: - name: elasticsearch - items: - - key: supervisord.conf - path: supervisord.conf - mode: 0644 - name: data - emptyDir: {} + hostPath: + path: /var/log/elasticsearch {{ end }} diff --git a/infra-charts/values.yaml.j2 b/infra-charts/values.yaml.j2 index 98c0752..b441ee6 100644 --- a/infra-charts/values.yaml.j2 +++ b/infra-charts/values.yaml.j2 @@ -76,38 +76,39 @@ danm_webhook: webhook_target_port: {{ caas.danm_webhook_target_port }} elasticsearch: -{% set clustermasternodes = [] -%} -{%- for masternumber in range(groups['caas_master']|length|int) -%} -{%- if clustermasternodes.append('elasticsearch-master-' + (masternumber|string)) -%}{%- endif -%} -{%- endfor %} - clustermasternodes: '{{ clustermasternodes|join(", ") }}' {% if caas.infra_log_store is defined and caas.infra_log_store != 'remote_syslog' %} required: true {% endif %} +{%- set clustermasternodes = [] -%} +{%- for masternumber in range(groups['caas_master']|length|int) -%} +{%- if clustermasternodes.append('elasticsearch-' + (masternumber|string)) -%}{%- endif -%} +{%- endfor -%} +{%- set cluster_master_nodes = clustermasternodes|join(",") %} url: {{ caas.elasticsearch_url }} port: {{ caas.elasticsearch_port }} scheme: {{ caas.elasticsearch_scheme }} logstash_prefix: {{ caas.elasticsearch_logstash_prefix }} image_name: {{ container_image_names | select('search', '/elasticsearch') | list | last }} memory_limit: {{ caas.elasticsearch_memory_limit }} - cpu_limit: 1 + cpu_limit: {{ caas.elasticsearch_cpu_limit }} memory_request: {{ caas.elasticsearch_memory_request }} + cpu_request: {{ caas.elasticsearch_cpu_request }} java_heap_request: {{ caas.elasticsearch_java_heap_request }} java_heap_max: {{ caas.elasticsearch_java_heap_max }} port_for_cluster_communication: {{ caas.elasticsearch_port_for_cluster_communication }} - es_uid: {{ caas.uid.elasticsearch }} cluster: name: "elasticsearch" +{% set minimum_master_node_num = (((groups['caas_master']|length|int)/2)|round(0, 'ceil'))|int %} +{% set master_node_num = groups['caas_master']|length|int %} env: -{% set minimummasternodes = ((((groups['caas_master']|length|int)/2)+1)|round(0, 'floor'))|int %} - MINIMUM_MASTER_NODES: {{ minimummasternodes }} - master: - name: elasticsearch-master - replicas: "{{ groups['caas_master']|length|int }}" - data: - name: elasticsearch-data - replicas: "{{ groups['caas_master']|length|int }}" - terminationGracePeriodSeconds: 3600 + DISCOVERY_SERVICE: {{ caas.elasticsearch_url }} + CLUSTER_INITIAL_MASTER_NODES: {{ cluster_master_nodes }} + EXPECTED_MASTER_NODES: {{ master_node_num }} + EXPECTED_DATA_NODES: {{ master_node_num }} + RECOVER_AFTER_MASTER_NODES: {{ minimum_master_node_num }} + RECOVER_AFTER_MASTER_NODES: {{ minimum_master_node_num }} + replicas: {{ master_node_num }} + terminationGracePeriodSeconds: 3600 sriovdp: required: true -- 2.16.6