Add support of ES intra-day data rotation

When filesystem (`/var/log`) usage exceeds 80%, a script starts
dropping the oldest ES index until fs usage goes below 80%.
If there is no more earlier-than-today indices left, this script
starts deleting oldest 100000 log entries to reclaim space as
last resort.

Signed-off-by: Ferenc Tóth <ferenc.2.toth@nokia.com>
Change-Id: I60940b38f896a440b5b0b0ecddbb9a6907341724

diff --git a/SPECS/infra-charts.spec b/SPECS/infra-charts.spec
index 385b5a5..aa8f193 100644 (file)
--- a/SPECS/infra-charts.spec
+++ b/SPECS/infra-charts.spec
@@ -15,7 +15,7 @@
 %define COMPONENT infra-charts
 %define RPM_NAME caas-%{COMPONENT}
 %define RPM_MAJOR_VERSION 1.0.0
-%define RPM_MINOR_VERSION 24
+%define RPM_MINOR_VERSION 25
 
 Name:           %{RPM_NAME}
 Version:        %{RPM_MAJOR_VERSION}

diff --git a/infra-charts/templates/elasticsearch-configmap.yaml b/infra-charts/templates/elasticsearch-configmap.yaml
index 92c128c..b8ef093 100644 (file)
--- a/infra-charts/templates/elasticsearch-configmap.yaml
+++ b/infra-charts/templates/elasticsearch-configmap.yaml
@@ -42,17 +42,6 @@ data:
     gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:3}
     gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:3}
 
-  log4j2.properties: |-
-    status = error
-    appender.console.type = Console
-    appender.console.name = console
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-    rootLogger.level = info
-    rootLogger.appenderRef.console.ref = console
-    logger.searchguard.name = com.floragunn
-    logger.searchguard.level = info
-
   supervisord.conf: |
     [unix_http_server]
     file=/run/supervisord.sock   ; (the path to the socket file)
@@ -61,8 +50,6 @@ data:
     username = dummy
     password = dummy
 
-    ;[inet_http_server]         ; inet (TCP) server disabled by default
-
     [supervisord]
     user=root
     logfile=/dev/null
@@ -103,7 +90,7 @@ data:
     stdout_logfile_maxbytes=0
 
     [program:setindex]
-    command=/usr/share/elasticsearch/elasticsearch-setindex
+    command=/usr/share/elasticsearch/elasticsearch-setindex.sh
     startsecs=3
     startretries=1
     stopwaitsecs = 3
@@ -111,30 +98,4 @@ data:
     redirect_stderr=true
     stdout_logfile=/dev/stdout
     stdout_logfile_maxbytes=0
-
-  elasticsearch-logrotate: |
-    #!/bin/bash
-    # Copyright 2019 Nokia
-    #
-    # Licensed under the Apache License, Version 2.0 (the "License");
-    # you may not use this file except in compliance with the License.
-    # You may obtain a copy of the License at
-    #
-    #     http://www.apache.org/licenses/LICENSE-2.0
-    #
-    # Unless required by applicable law or agreed to in writing, software
-    # distributed under the License is distributed on an "AS IS" BASIS,
-    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    # See the License for the specific language governing permissions and
-    # limitations under the License.
-
-    FS_LIMIT=80
-    declare -i pcent=100
-    while [ $pcent -ge $FS_LIMIT ]
-    do
-        pcent=`df --output=pcent /usr/share/elasticsearch/data | tail -n1 | tr -d '%'`
-        if [ $pcent -le $FS_LIMIT ]; then exit; fi
-        oldest=`curl -s -XGET 'localhost:{{ .Values.elasticsearch.port }}/_cat/indices' | awk '{print $3}' | egrep '^.+-[[:digit:]]{4}\.[[:digit:]]{2}\.[[:digit:]]{2}$' | sort -t'-' -k2 | head -n1`
-        curl -XDELETE "localhost:{{ .Values.elasticsearch.port }}/$oldest" >/dev/null
-    done
 {{ end }}

diff --git a/infra-charts/templates/elasticsearch-data-statefulset.yaml b/infra-charts/templates/elasticsearch-data-statefulset.yaml
index d9c8326..afc949a 100644 (file)
--- a/infra-charts/templates/elasticsearch-data-statefulset.yaml
+++ b/infra-charts/templates/elasticsearch-data-statefulset.yaml
@@ -89,11 +89,13 @@ spec:
             cpu: "100m"
             memory: "{{ .Values.elasticsearch.memory_request }}"
         readinessProbe:
-          tcpSocket:
+          httpGet:
+            path: /_cluster/health?local=true
             port: {{ .Values.elasticsearch.port }}
           initialDelaySeconds: 5
         livenessProbe:
-          tcpSocket:
+          httpGet:
+            path: /_cluster/health?local=true
             port: {{ .Values.elasticsearch.port }}
           initialDelaySeconds: 60
           timeoutSeconds: 5
@@ -111,9 +113,6 @@ spec:
         - mountPath: /etc/supervisord.conf
           name: supervisord-config
           subPath: supervisord.conf
-        - mountPath: /etc/cron.hourly/elasticsearch-logrotate
-          name: elasticsearch-logrotate
-          subPath: elasticsearch-logrotate
       terminationGracePeriodSeconds: {{ .Values.elasticsearch.data.terminationGracePeriodSeconds }}
       volumes:
         - name: time-mount
@@ -133,13 +132,6 @@ spec:
               - key: supervisord.conf
                 path: supervisord.conf
                 mode: 0644
-        - name: elasticsearch-logrotate
-          configMap:
-            name: elasticsearch
-            items:
-              - key: elasticsearch-logrotate
-                path: elasticsearch-logrotate
-                mode: 0755
         - name: data
           hostPath:
             path: /var/log/elasticsearch

diff --git a/infra-charts/templates/elasticsearch-master-statefulset.yaml b/infra-charts/templates/elasticsearch-master-statefulset.yaml
index 30ba145..cf3d274 100644 (file)
--- a/infra-charts/templates/elasticsearch-master-statefulset.yaml
+++ b/infra-charts/templates/elasticsearch-master-statefulset.yaml
@@ -86,7 +86,8 @@ spec:
             port: {{ .Values.elasticsearch.port }}
           initialDelaySeconds: 5
         livenessProbe:
-          tcpSocket:
+          httpGet:
+            path: /_cluster/health?local=true
             port: {{ .Values.elasticsearch.port }}
           initialDelaySeconds: 60
           timeoutSeconds: 5