From 454cbff6be17bd8ef6cc43d71ceecabc3a7af80e Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ferenc=20T=C3=B3th?= Date: Mon, 9 Sep 2019 15:09:58 +0200 Subject: [PATCH] Add support of ES intra-day data rotation MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit When filesystem (`/var/log`) usage exceeds 80%, a script starts dropping the oldest ES index until fs usage goes below 80%. If there is no more earlier-than-today indices left, this script starts deleting oldest 100000 log entries to reclaim space as last resort. Signed-off-by: Ferenc Tóth Change-Id: I60940b38f896a440b5b0b0ecddbb9a6907341724 --- SPECS/infra-charts.spec | 2 +- .../templates/elasticsearch-configmap.yaml | 41 +--------------------- .../templates/elasticsearch-data-statefulset.yaml | 16 +++------ .../elasticsearch-master-statefulset.yaml | 3 +- 4 files changed, 8 insertions(+), 54 deletions(-) diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec index 385b5a5..aa8f193 100644 --- a/SPECS/infra-charts.spec +++ b/SPECS/infra-charts.spec @@ -15,7 +15,7 @@ %define COMPONENT infra-charts %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 1.0.0 -%define RPM_MINOR_VERSION 24 +%define RPM_MINOR_VERSION 25 Name: %{RPM_NAME} Version: %{RPM_MAJOR_VERSION} diff --git a/infra-charts/templates/elasticsearch-configmap.yaml b/infra-charts/templates/elasticsearch-configmap.yaml index 92c128c..b8ef093 100644 --- a/infra-charts/templates/elasticsearch-configmap.yaml +++ b/infra-charts/templates/elasticsearch-configmap.yaml @@ -42,17 +42,6 @@ data: gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:3} gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:3} - log4j2.properties: |- - status = error - appender.console.type = Console - appender.console.name = console - appender.console.layout.type = PatternLayout - appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - rootLogger.level = info - rootLogger.appenderRef.console.ref = console - logger.searchguard.name = com.floragunn - logger.searchguard.level = info - supervisord.conf: | [unix_http_server] file=/run/supervisord.sock ; (the path to the socket file) @@ -61,8 +50,6 @@ data: username = dummy password = dummy - ;[inet_http_server] ; inet (TCP) server disabled by default - [supervisord] user=root logfile=/dev/null @@ -103,7 +90,7 @@ data: stdout_logfile_maxbytes=0 [program:setindex] - command=/usr/share/elasticsearch/elasticsearch-setindex + command=/usr/share/elasticsearch/elasticsearch-setindex.sh startsecs=3 startretries=1 stopwaitsecs = 3 @@ -111,30 +98,4 @@ data: redirect_stderr=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 - - elasticsearch-logrotate: | - #!/bin/bash - # Copyright 2019 Nokia - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - FS_LIMIT=80 - declare -i pcent=100 - while [ $pcent -ge $FS_LIMIT ] - do - pcent=`df --output=pcent /usr/share/elasticsearch/data | tail -n1 | tr -d '%'` - if [ $pcent -le $FS_LIMIT ]; then exit; fi - oldest=`curl -s -XGET 'localhost:{{ .Values.elasticsearch.port }}/_cat/indices' | awk '{print $3}' | egrep '^.+-[[:digit:]]{4}\.[[:digit:]]{2}\.[[:digit:]]{2}$' | sort -t'-' -k2 | head -n1` - curl -XDELETE "localhost:{{ .Values.elasticsearch.port }}/$oldest" >/dev/null - done {{ end }} diff --git a/infra-charts/templates/elasticsearch-data-statefulset.yaml b/infra-charts/templates/elasticsearch-data-statefulset.yaml index d9c8326..afc949a 100644 --- a/infra-charts/templates/elasticsearch-data-statefulset.yaml +++ b/infra-charts/templates/elasticsearch-data-statefulset.yaml @@ -89,11 +89,13 @@ spec: cpu: "100m" memory: "{{ .Values.elasticsearch.memory_request }}" readinessProbe: - tcpSocket: + httpGet: + path: /_cluster/health?local=true port: {{ .Values.elasticsearch.port }} initialDelaySeconds: 5 livenessProbe: - tcpSocket: + httpGet: + path: /_cluster/health?local=true port: {{ .Values.elasticsearch.port }} initialDelaySeconds: 60 timeoutSeconds: 5 @@ -111,9 +113,6 @@ spec: - mountPath: /etc/supervisord.conf name: supervisord-config subPath: supervisord.conf - - mountPath: /etc/cron.hourly/elasticsearch-logrotate - name: elasticsearch-logrotate - subPath: elasticsearch-logrotate terminationGracePeriodSeconds: {{ .Values.elasticsearch.data.terminationGracePeriodSeconds }} volumes: - name: time-mount @@ -133,13 +132,6 @@ spec: - key: supervisord.conf path: supervisord.conf mode: 0644 - - name: elasticsearch-logrotate - configMap: - name: elasticsearch - items: - - key: elasticsearch-logrotate - path: elasticsearch-logrotate - mode: 0755 - name: data hostPath: path: /var/log/elasticsearch diff --git a/infra-charts/templates/elasticsearch-master-statefulset.yaml b/infra-charts/templates/elasticsearch-master-statefulset.yaml index 30ba145..cf3d274 100644 --- a/infra-charts/templates/elasticsearch-master-statefulset.yaml +++ b/infra-charts/templates/elasticsearch-master-statefulset.yaml @@ -86,7 +86,8 @@ spec: port: {{ .Values.elasticsearch.port }} initialDelaySeconds: 5 livenessProbe: - tcpSocket: + httpGet: + path: /_cluster/health?local=true port: {{ .Values.elasticsearch.port }} initialDelaySeconds: 60 timeoutSeconds: 5 -- 2.16.6