From: Ferenc Tóth Date: Mon, 9 Sep 2019 14:19:24 +0000 (+0200) Subject: Add ES data rotate & cleanup X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fcaas-logging.git;a=commitdiff_plain;h=dbb52626787b945265e4dbfaee8a2905fa410b67 Add ES data rotate & cleanup - `bom.json` files -> not used - `config/elastic/elasticsearch.yml` -> already moved to configmap - `config/logrotate` -> no ES logs to rotate - `healthcheck` -> not used - disable Java GC logging Signed-off-by: Ferenc Tóth Change-Id: I37ed84dd1fd09e9cef070da1895e83f02c4be4a2 --- diff --git a/SPECS/elasticsearch.spec b/SPECS/elasticsearch.spec index 11d4644..45c19df 100644 --- a/SPECS/elasticsearch.spec +++ b/SPECS/elasticsearch.spec @@ -15,7 +15,7 @@ %define COMPONENT elasticsearch %define RPM_NAME caas-%{COMPONENT} %define RPM_MAJOR_VERSION 7.3.0 -%define RPM_MINOR_VERSION 0 +%define RPM_MINOR_VERSION 1 %define IMAGE_TAG %{RPM_MAJOR_VERSION}-%{RPM_MINOR_VERSION} Name: %{RPM_NAME} diff --git a/docker-build/elasticsearch/Dockerfile b/docker-build/elasticsearch/Dockerfile index ad8edab..39f8ecb 100644 --- a/docker-build/elasticsearch/Dockerfile +++ b/docker-build/elasticsearch/Dockerfile @@ -37,29 +37,13 @@ RUN groupadd -g 202 elasticsearch \ supervisor \ && yum clean all \ && curl -fsSL ${ES_TARBAL} | tar zx --strip-components=1 \ -&& echo "===> Creating Elasticsearch Paths..." \ - && for path in \ - /usr/share/elasticsearch/data \ - /usr/share/elasticsearch/logs \ - /usr/share/elasticsearch/config \ - /usr/share/elasticsearch/config/scripts \ - /usr/share/elasticsearch/plugins \ - ; do \ - mkdir -p "$path"; \ - chown -R elasticsearch:elasticsearch "$path"; \ - done - -COPY config/elastic /usr/share/elasticsearch/config -COPY config/logrotate /etc/logrotate.d/elasticsearch -COPY healthcheck /usr/local/bin/ -COPY elasticsearch-setindex /usr/share/elasticsearch - -RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch /etc/logrotate.d/elasticsearch /usr/local/bin/healthcheck \ -&& chmod +x /usr/share/elasticsearch/elasticsearch-setindex \ +&& chown -R elasticsearch:elasticsearch /usr/share/elasticsearch \ # workaround for Elasticsearch: this way ES can list the used plugins, and remove the jdk folder (open jdk slimmer) && cp -al /usr/share/elasticsearch/modules/* /usr/share/elasticsearch/plugins/ \ -&& chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/plugins \ -&& rm -rf /usr/share/elasticsearch/jdk +&& rm -rf /usr/share/elasticsearch/jdk \ # workaround end +&& sed '/GC logging/Q' -i /usr/share/elasticsearch/config/jvm.options # disable GC logging -CMD ["elasticsearch"] +COPY elasticsearch-logrotate.sh /etc/cron.hourly/ +COPY elasticsearch-setindex.sh /usr/share/elasticsearch/ +COPY log4j2.properties /usr/share/elasticsearch/config/ diff --git a/docker-build/elasticsearch/bom.json b/docker-build/elasticsearch/bom.json deleted file mode 100644 index d2ffdd2..0000000 --- a/docker-build/elasticsearch/bom.json +++ /dev/null @@ -1,1030 +0,0 @@ -{ - "bom": [ - { - "name": "centos-linux-container", - "version": "7.6.1810", - "source-url": "https://github.com/CentOS/sig-cloud-instance-images/archive/7c2e214edced0b2f22e663ab4175a80fc93acaa9.tar.gz", - "foss": "yes" - }, - { - "name": "acl.x86_64", - "version": "2.2.51-14.el7", - "source-url": "purkki.dynamic.nsn-net.net/mirrors/centos/snapshot/20181003/7/os/Source/SPackages/acl-2.2.51-14.el7.src.rpm", - "foss": "yes" - }, - { - "name": "audit-libs.x86_64", - "version": "2.8.4-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/audit-2.8.4-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "basesystem.noarch", - "version": "10.0-7.el7.centos", - "source-url": "http://purkki.dynamic.nsn-net.net/mirrors/centos/snapshot/20181003/7/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm", - "foss": "yes" - }, - { - "name": "bash.x86_64", - "version": "4.2.46-31.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/bash-4.2.46-31.el7.src.rpm", - "foss": "yes" - }, - { - "name": "bind-license.noarch", - "version": "32:9.9.4-72.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/bind-9.9.4-72.el7.src.rpm", - "foss": "yes" - }, - { - "name": "binutils.x86_64", - "version": "2.27-34.base.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/binutils-2.27-34.base.el7.src.rpm", - "foss": "yes" - }, - { - "name": "bzip2.x86_64", - "version": "1.0.6-13.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/bzip2-1.0.6-13.el7.src.rpm", - "foss": "yes" - }, - { - "name": "bzip2-libs.x86_64", - "version": "1.0.6-13.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/bzip2-1.0.6-13.el7.src.rpm", - "foss": "yes" - }, - { - "name": "ca-certificates.noarch", - "version": "2018.2.22-70.0.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/ca-certificates-2018.2.22-70.0.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "centos-release.x86_64", - "version": "7-6.1810.2.el7.centos", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/centos-release-7-6.1810.2.el7.centos.src.rpm", - "foss": "yes" - }, - { - "name": "chkconfig.x86_64", - "version": "1.7.4-1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/chkconfig-1.7.4-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "copy-jdk-configs.noarch", - "version": "3.3-10.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/copy-jdk-configs-3.3-10.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "coreutils.x86_64", - "version": "8.22-23.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/coreutils-8.22-23.el7.src.rpm", - "foss": "yes" - }, - { - "name": "cpio.x86_64", - "version": "2.11-27.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/cpio-2.11-27.el7.src.rpm", - "foss": "yes" - }, - { - "name": "cracklib.x86_64", - "version": "2.9.0-11.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/cracklib-2.9.0-11.el7.src.rpm", - "foss": "yes" - }, - { - "name": "cracklib-dicts.x86_64", - "version": "2.9.0-11.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/cracklib-2.9.0-11.el7.src.rpm", - "foss": "yes" - }, - { - "name": "cronie.x86_64", - "version": "1.4.11-20.el7_6", - "source-url": "vault.centos.org/7.6.1810/updates/Source/SPackages/cronie-1.4.11-20.el7_6.src.rpm", - "foss": "yes" - }, - { - "name": "cronie-anacron.x86_64", - "version": "1.4.11-20.el7_6", - "source-url": "vault.centos.org/7.6.1810/updates/Source/SPackages/cronie-1.4.11-20.el7_6.src.rpm", - "foss": "yes" - }, - { - "name": "crontabs.noarch", - "version": "1.11-6.20121102git.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/crontabs-1.11-6.20121102git.el7.src.rpm", - "foss": "yes" - }, - { - "name": "cryptsetup-libs.x86_64", - "version": "2.0.3-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/cryptsetup-2.0.3-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "curl.x86_64", - "version": "7.29.0-51.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/curl-7.29.0-51.el7.src.rpm", - "foss": "yes" - }, - { - "name": "cyrus-sasl-lib.x86_64", - "version": "2.1.26-23.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/cyrus-sasl-2.1.26-23.el7.src.rpm", - "foss": "yes" - }, - { - "name": "dbus.x86_64", - "version": "1:1.10.24-12.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/dbus-1.10.24-12.el7.src.rpm", - "foss": "yes" - }, - { - "name": "dbus-glib.x86_64", - "version": "0.100-7.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/dbus-glib-0.100-7.el7.src.rpm", - "foss": "yes" - }, - { - "name": "dbus-libs.x86_64", - "version": "1:1.10.24-12.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/dbus-1.10.24-12.el7.src.rpm", - "foss": "yes" - }, - { - "name": "dbus-python.x86_64", - "version": "1.1.1-9.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/dbus-python-1.1.1-9.el7.src.rpm", - "foss": "yes" - }, - { - "name": "device-mapper.x86_64", - "version": "7:1.02.149-8.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/lvm2-2.02.180-8.el7.src.rpm", - "foss": "yes" - }, - { - "name": "device-mapper-libs.x86_64", - "version": "7:1.02.149-8.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/lvm2-2.02.180-8.el7.src.rpm", - "foss": "yes" - }, - { - "name": "diffutils.x86_64", - "version": "3.3-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/diffutils-3.3-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "dracut.x86_64", - "version": "033-554.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/dracut-033-554.el7.src.rpm", - "foss": "yes" - }, - { - "name": "elfutils-default-yama-scope.noarch", - "version": "0.172-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/elfutils-0.172-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "elfutils-libelf.x86_64", - "version": "0.172-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/elfutils-0.172-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "elfutils-libs.x86_64", - "version": "0.172-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/elfutils-0.172-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "expat.x86_64", - "version": "2.1.0-10.el7_3", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/expat-2.1.0-10.el7_3.src.rpm", - "foss": "yes" - }, - { - "name": "file-libs.x86_64", - "version": "5.11-35.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/file-5.11-35.el7.src.rpm", - "foss": "yes" - }, - { - "name": "filesystem.x86_64", - "version": "3.2-25.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/filesystem-3.2-25.el7.src.rpm", - "foss": "yes" - }, - { - "name": "findutils.x86_64", - "version": "1:4.5.11-6.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/findutils-4.5.11-6.el7.src.rpm", - "foss": "yes" - }, - { - "name": "freetype.x86_64", - "version": "2.8-12.el7_6.1", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/freetype-2.8-12.el7.src.rpm", - "foss": "yes" - }, - { - "name": "gawk.x86_64", - "version": "4.0.2-4.el7_3.1", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gawk-4.0.2-4.el7_3.1.src.rpm", - "foss": "yes" - }, - { - "name": "gdbm.x86_64", - "version": "1.10-8.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gdbm-1.10-8.el7.src.rpm", - "foss": "yes" - }, - { - "name": "glib2.x86_64", - "version": "2.56.1-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/glib2-2.56.1-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "glibc.x86_64", - "version": "2.17-260.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/glibc-2.17-260.el7.src.rpm", - "foss": "yes" - }, - { - "name": "glibc-common.x86_64", - "version": "2.17-260.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/glibc-2.17-260.el7.src.rpm", - "foss": "yes" - }, - { - "name": "gmp.x86_64", - "version": "1:6.0.0-15.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gmp-6.0.0-15.el7.src.rpm", - "foss": "yes" - }, - { - "name": "gnupg2.x86_64", - "version": "2.0.22-5.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gnupg2-2.0.22-5.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "gobject-introspection.x86_64", - "version": "1.56.1-1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gobject-introspection-1.56.1-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "gpgme.x86_64", - "version": "1.3.2-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gpgme-1.3.2-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "grep.x86_64", - "version": "2.20-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/grep-2.20-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "gzip.x86_64", - "version": "1.5-10.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gzip-1.5-10.el7.src.rpm", - "foss": "yes" - }, - { - "name": "hardlink.x86_64", - "version": "1:1.0-19.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/hardlink-1.0-19.el7.src.rpm", - "foss": "yes" - }, - { - "name": "hostname.x86_64", - "version": "3.13-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/hostname-3.13-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "info.x86_64", - "version": "5.1-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/texinfo-5.1-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "iputils.x86_64", - "version": "20160308-10.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/iputils-20160308-10.el7.src.rpm", - "foss": "yes" - }, - { - "name": "java-1.8.0-openjdk-headless.x86_64", - "version": "1:1.8.0.191.b12-1.el7_6", - "source-url": "vault.centos.org/7.6.1810/updates/Source/SPackages/java-1.8.0-openjdk-1.8.0.191.b12-1.el7_6.src.rpm", - "foss": "yes" - }, - { - "name": "javapackages-tools.noarch", - "version": "3.4.1-11.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/javapackages-tools-3.4.1-11.el7.src.rpm", - "foss": "yes" - }, - { - "name": "json-c.x86_64", - "version": "0.11-4.el7_0", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/json-c-0.11-4.el7_0.src.rpm", - "foss": "yes" - }, - { - "name": "keyutils-libs.x86_64", - "version": "1.5.8-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/keyutils-1.5.8-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "kmod.x86_64", - "version": "20-23.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/kmod-20-23.el7.src.rpm", - "foss": "yes" - }, - { - "name": "kmod-libs.x86_64", - "version": "20-23.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/kmod-20-23.el7.src.rpm", - "foss": "yes" - }, - { - "name": "kpartx.x86_64", - "version": "0.4.9-123.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/device-mapper-multipath-0.4.9-123.el7.src.rpm", - "foss": "yes" - }, - { - "name": "krb5-libs.x86_64", - "version": "1.15.1-34.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/krb5-1.15.1-34.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libacl.x86_64", - "version": "2.2.51-14.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/acl-2.2.51-14.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libassuan.x86_64", - "version": "2.1.0-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libassuan-2.1.0-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libattr.x86_64", - "version": "2.4.46-13.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/attr-2.4.46-13.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libblkid.x86_64", - "version": "2.23.2-59.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/util-linux-2.23.2-59.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libcap.x86_64", - "version": "2.22-9.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libcap-2.22-9.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libcap-ng.x86_64", - "version": "0.7.5-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libcap-ng-0.7.5-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libcom_err.x86_64", - "version": "1.42.9-13.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/e2fsprogs-1.42.9-13.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libcurl.x86_64", - "version": "7.29.0-51.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/curl-7.29.0-51.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libdb.x86_64", - "version": "5.3.21-24.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libdb-5.3.21-24.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libdb-utils.x86_64", - "version": "5.3.21-24.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libdb-5.3.21-24.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libffi.x86_64", - "version": "3.0.13-18.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libffi-3.0.13-18.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libgcc.x86_64", - "version": "4.8.5-36.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gcc-4.8.5-36.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libgcrypt.x86_64", - "version": "1.5.3-14.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libgcrypt-1.5.3-14.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libgpg-error.x86_64", - "version": "1.12-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libgpg-error-1.12-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libidn.x86_64", - "version": "1.28-4.el7", - "source-url": "entos.org/7.6.1810/os/Source/SPackages/libidn-1.28-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libjpeg-turbo.x86_64", - "version": "1.2.90-6.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libjpeg-turbo-1.2.90-6.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libmount.x86_64", - "version": "2.23.2-59.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/util-linux-2.23.2-59.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libpng.x86_64", - "version": "2:1.5.13-7.el7_2", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libpng-1.5.13-7.el7_2.src.rpm", - "foss": "yes" - }, - { - "name": "libpwquality.x86_64", - "version": "1.2.3-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libpwquality-1.2.3-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libselinux.x86_64", - "version": "2.5-14.1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libselinux-2.5-14.1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libsemanage.x86_64", - "version": "2.5-14.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libsemanage-2.5-14.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libsepol.x86_64", - "version": "2.5-10.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libsepol-2.5-10.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libsmartcols.x86_64", - "version": "2.23.2-59.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/util-linux-2.23.2-59.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libssh2.x86_64", - "version": "1.4.3-12.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libssh2-1.4.3-12.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libstdc++.x86_64", - "version": "4.8.5-36.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/gcc-4.8.5-36.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libtasn1.x86_64", - "version": "4.10-1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libtasn1-4.10-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libuser.x86_64", - "version": "0.60-9.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libuser-0.60-9.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libutempter.x86_64", - "version": "1.1.6-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libutempter-1.1.6-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libuuid.x86_64", - "version": "2.23.2-59.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/util-linux-2.23.2-59.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libverto.x86_64", - "version": "0.2.5-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libverto-0.2.5-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "libxml2.x86_64", - "version": "2.9.1-6.el7_2.3", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libxml2-2.9.1-6.el7_2.3.src.rpm", - "foss": "yes" - }, - { - "name": "libxml2-python.x86_64", - "version": "2.9.1-6.el7_2.3", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libxml2-2.9.1-6.el7_2.3.src.rpm", - "foss": "yes" - }, - { - "name": "libxslt.x86_64", - "version": "1.1.28-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/libxslt-1.1.28-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "lksctp-tools.x86_64", - "version": "1.0.17-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/lksctp-tools-1.0.17-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "lua.x86_64", - "version": "5.1.4-15.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/lua-5.1.4-15.el7.src.rpm", - "foss": "yes" - }, - { - "name": "lz4.x86_64", - "version": "1.7.5-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/lz4-1.7.5-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "ncurses.x86_64", - "version": "5.9-14.20130511.el7_4", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/ncurses-5.9-14.20130511.el7_4.src.rpm", - "foss": "yes" - }, - { - "name": "ncurses-base.noarch", - "version": "5.9-14.20130511.el7_4", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/ncurses-5.9-14.20130511.el7_4.src.rpm", - "foss": "yes" - }, - { - "name": "ncurses-libs.x86_64", - "version": "5.9-14.20130511.el7_4", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/ncurses-5.9-14.20130511.el7_4.src.rpm", - "foss": "yes" - }, - { - "name": "nspr.x86_64", - "version": "4.19.0-1.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nspr-4.19.0-1.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "nss.x86_64", - "version": "3.36.0-7.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-3.36.0-7.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "nss-pem.x86_64", - "version": "1.0.3-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-pem-1.0.3-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "nss-softokn.x86_64", - "version": "3.36.0-5.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-softokn-3.36.0-5.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "nss-softokn-freebl.x86_64", - "version": "3.36.0-5.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-softokn-3.36.0-5.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "nss-sysinit.x86_64", - "version": "3.36.0-7.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-softokn-3.36.0-5.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "nss-tools.x86_64", - "version": "3.36.0-7.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-softokn-3.36.0-5.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "nss-util.x86_64", - "version": "3.36.0-1.el7_5", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/nss-softokn-3.36.0-5.el7_5.src.rpm", - "foss": "yes" - }, - { - "name": "openldap.x86_64", - "version": "2.4.44-20.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/openldap-2.4.44-20.el7.src.rpm", - "foss": "yes" - }, - { - "name": "openssl-libs.x86_64", - "version": "1:1.0.2k-16.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/openssl-1.0.2k-16.el7.src.rpm", - "foss": "yes" - }, - { - "name": "p11-kit.x86_64", - "version": "0.23.5-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/p11-kit-0.23.5-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "p11-kit-trust.x86_64", - "version": "0.23.5-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/p11-kit-0.23.5-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pam.x86_64", - "version": "1.1.8-22.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pam-1.1.8-22.el7.src.rpm", - "foss": "yes" - }, - { - "name": "passwd.x86_64", - "version": "0.79-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/passwd-0.79-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pcre.x86_64", - "version": "8.32-17.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pcre-8.32-17.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pinentry.x86_64", - "version": "0.8.1-17.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pinentry-0.8.1-17.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pkgconfig.x86_64", - "version": "1:0.27.1-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pkgconfig-0.27.1-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "popt.x86_64", - "version": "1.13-16.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/popt-1.13-16.el7.src.rpm", - "foss": "yes" - }, - { - "name": "procps-ng.x86_64", - "version": "3.3.10-23.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/procps-ng-3.3.10-23.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pth.x86_64", - "version": "2.0.7-23.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pth-2.0.7-23.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pygpgme.x86_64", - "version": "0.3-9.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pygpgme-0.3-9.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pyliblzma.x86_64", - "version": "0.5.3-11.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pyliblzma-0.5.3-11.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python.x86_64", - "version": "2.7.5-76.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-2.7.5-76.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-backports.x86_64", - "version": "1.0-8.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-backports-1.0-8.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-backports-ssl_match_hostname.noarch", - "version": "3.5.0.1-1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-backports-ssl_match_hostname-3.5.0.1-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-chardet.noarch", - "version": "2.2.1-1.el7_1", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-chardet-2.2.1-1.el7_1.src.rpm", - "foss": "yes" - }, - { - "name": "python-gobject-base.x86_64", - "version": "3.22.0-1.el7_4.1", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pygobject3-3.22.0-1.el7_4.1.src.rpm", - "foss": "yes" - }, - { - "name": "python-iniparse.noarch", - "version": "0.4-9.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-iniparse-0.4-9.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-ipaddress.noarch", - "version": "1.0.16-2.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-ipaddress-1.0.16-2.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-javapackages.noarch", - "version": "3.4.1-11.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/javapackages-tools-3.4.1-11.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-kitchen.noarch", - "version": "1.1.1-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-kitchen-1.1.1-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-libs.x86_64", - "version": "2.7.5-76.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-2.7.5-76.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-lxml.x86_64", - "version": "3.2.1-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-lxml-3.2.1-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-meld3.x86_64", - "version": "0.6.10-1.el7", - "source-url": "https://download-ib01.fedoraproject.org/pub/epel/7/SRPMS/Packages/p/python-meld3-0.6.10-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-pycurl.x86_64", - "version": "7.19.0-19.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-pycurl-7.19.0-19.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-setuptools.noarch", - "version": "0.9.8-7.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-setuptools-0.9.8-7.el7.src.rpm", - "foss": "yes" - }, - { - "name": "python-urlgrabber.noarch", - "version": "3.10-9.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/python-urlgrabber-3.10-9.el7.src.rpm", - "foss": "yes" - }, - { - "name": "pyxattr.x86_64", - "version": "0.5.1-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/pyxattr-0.5.1-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "qrencode-libs.x86_64", - "version": "3.4.1-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/qrencode-3.4.1-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "readline.x86_64", - "version": "6.2-10.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/readline-6.2-10.el7.src.rpm", - "foss": "yes" - }, - { - "name": "rootfiles.noarch", - "version": "8.1-11.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm", - "foss": "yes" - }, - { - "name": "rpm.x86_64", - "version": "4.11.3-35.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/rpm-4.11.3-35.el7.src.rpm", - "foss": "yes" - }, - { - "name": "rpm-build-libs.x86_64", - "version": "4.11.3-35.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/rpm-4.11.3-35.el7.src.rpm", - "foss": "yes" - }, - { - "name": "rpm-libs.x86_64", - "version": "4.11.3-35.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/rpm-4.11.3-35.el7.src.rpm", - "foss": "yes" - }, - { - "name": "rpm-python.x86_64", - "version": "4.11.3-35.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/rpm-4.11.3-35.el7.src.rpm", - "foss": "yes" - }, - { - "name": "sed.x86_64", - "version": "4.2.2-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/sed-4.2.2-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "setup.noarch", - "version": "2.8.71-10.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/setup-2.8.71-10.el7.src.rpm", - "foss": "yes" - }, - { - "name": "shadow-utils.x86_64", - "version": "2:4.1.5.1-25.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/shadow-utils-4.1.5.1-25.el7.src.rpm", - "foss": "yes" - }, - { - "name": "shared-mime-info.x86_64", - "version": "1.8-4.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/shared-mime-info-1.8-4.el7.src.rpm", - "foss": "yes" - }, - { - "name": "sqlite.x86_64", - "version": "3.7.17-8.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/sqlite-3.7.17-8.el7.src.rpm", - "foss": "yes" - }, - { - "name": "supervisor.noarch", - "version": "3.1.4-1.el7", - "source-url": "https://download-ib01.fedoraproject.org/pub/epel/7/SRPMS/Packages/s/supervisor-3.1.4-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "systemd.x86_64", - "version": "219-62.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/systemd-219-62.el7.src.rpm", - "foss": "yes" - }, - { - "name": "systemd-libs.x86_64", - "version": "219-62.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/systemd-219-62.el7.src.rpm", - "foss": "yes" - }, - { - "name": "tar.x86_64", - "version": "2:1.26-35.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/tar-1.26-35.el7.src.rpm", - "foss": "yes" - }, - { - "name": "tzdata.noarch", - "version": "2018e-3.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/tzdata-2018e-3.el7.src.rpm", - "foss": "yes" - }, - { - "name": "tzdata-java.noarch", - "version": "2018i-1.el7", - "source-url": "vault.centos.org/7.6.1810/updates/Source/SPackages/tzdata-2018i-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "unzip.x86_64", - "version": "6.0-19.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/unzip-6.0-19.el7.src.rpm", - "foss": "yes" - }, - { - "name": "ustr.x86_64", - "version": "1.0.4-16.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/ustr-1.0.4-16.el7.src.rpm", - "foss": "yes" - }, - { - "name": "util-linux.x86_64", - "version": "2.23.2-59.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/util-linux-2.23.2-59.el7.src.rpm", - "foss": "yes" - }, - { - "name": "vim-minimal.x86_64", - "version": "2:7.4.160-5.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/vim-7.4.160-5.el7.src.rpm", - "foss": "yes" - }, - { - "name": "which.x86_64", - "version": "2.20-7.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/which-2.20-7.el7.src.rpm", - "foss": "yes" - }, - { - "name": "xz.x86_64", - "version": "5.2.2-1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/xz-5.2.2-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "xz-libs.x86_64", - "version": "5.2.2-1.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/xz-5.2.2-1.el7.src.rpm", - "foss": "yes" - }, - { - "name": "yum.noarch", - "version": "3.4.3-161.el7.centos", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/yum-3.4.3-161.el7.centos.src.rpm", - "foss": "yes" - }, - { - "name": "yum-metadata-parser.x86_64", - "version": "1.1.4-10.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/yum-metadata-parser-1.1.4-10.el7.src.rpm", - "foss": "yes" - }, - { - "name": "yum-plugin-fastestmirror.noarch", - "version": "1.1.31-50.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/yum-utils-1.1.31-50.el7.src.rpm", - "foss": "yes" - }, - { - "name": "yum-plugin-ovl.noarch", - "version": "1.1.31-50.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/yum-utils-1.1.31-50.el7.src.rpm", - "foss": "yes" - }, - { - "name": "yum-utils.noarch", - "version": "1.1.31-50.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/yum-utils-1.1.31-50.el7.src.rpm", - "foss": "yes" - }, - { - "name": "zlib.x86_64", - "version": "1.2.7-18.el7", - "source-url": "vault.centos.org/7.6.1810/os/Source/SPackages/zlib-1.2.7-18.el7.src.rpm", - "foss": "yes" - }, - { - "name": "elasticsearch", - "version": "6.6.1", - "source-url": "https://github.com/elastic/elasticsearch/archive/v6.6.1.tar.gz", - "foss": "yes" - } - ] -} diff --git a/docker-build/elasticsearch/config/elastic/elasticsearch.yml b/docker-build/elasticsearch/config/elastic/elasticsearch.yml deleted file mode 100644 index dc37e10..0000000 --- a/docker-build/elasticsearch/config/elastic/elasticsearch.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Copyright 2019 Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -network.host: 0.0.0.0 - -# this value is required because we set "network.host" -# be sure to modify it appropriately for a production cluster deployment -discovery.zen.minimum_master_nodes: 1 -# bootstrap.memory_lock: true - -node.master: true -node.ingest: true -node.data: true diff --git a/docker-build/elasticsearch/config/logrotate b/docker-build/elasticsearch/config/logrotate deleted file mode 100644 index 97f95a2..0000000 --- a/docker-build/elasticsearch/config/logrotate +++ /dev/null @@ -1,11 +0,0 @@ -/var/log/elasticsearch/*.log { - daily - rotate 50 - size 50M - copytruncate - compress - delaycompress - missingok - notifempty - create 644 elasticsearch elasticsearch -} diff --git a/docker-build/elasticsearch/elasticsearch-logrotate.sh b/docker-build/elasticsearch/elasticsearch-logrotate.sh new file mode 100755 index 0000000..989aedb --- /dev/null +++ b/docker-build/elasticsearch/elasticsearch-logrotate.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FS_LIMIT=80 +DOCS_DROP=100000 +ES_URL="http://localhost:$ELASTICSEARCH_LOGGING_SERVICE_PORT" + +log () { + echo "LOGROTATE: $*" >/proc/1/fd/1 +} + +log "hourly job started" + +IFS='-' read es type num <<< "$HOSTNAME" +if [[ "$type" != "data" ]]; then + log "non-data node -> exiting" + exit 0 +fi + +# sleep to avoid concurrent runs across multiple ES data Pods +let "SLEEP=($num * 600)+($RANDOM % 30)" +log "sleeping $SLEEP seconds..." +sleep $SLEEP + +TODAY=`date -u +%Y.%m.%d` +declare -i pcent=100 +while [ $pcent -ge $FS_LIMIT ] +do + pcent=`df --output=pcent /usr/share/elasticsearch/data | tail -n1 | tr -d '%'` + log "current filesystem usage: $pcent%" + if [ $pcent -le $FS_LIMIT ]; then break; fi + index_drop=`curl -sS -XGET "$ES_URL/_cat/indices?h=index" | egrep '^.+-[[:digit:]]{4}\.[[:digit:]]{2}\.[[:digit:]]{2}$' | grep -v "$TODAY" | sort -t'-' -k2 | head -n1` + if [ -n "$index_drop" ]; then + log "drop index: $index_drop" + curl -sS -XDELETE "$ES_URL/$index_drop" >/dev/null + sleep 5 + else + log "drop oldest $DOCS_DROP log entries" + curl -sS -XPOST "$ES_URL/*-$TODAY/_delete_by_query?sort=@timestamp:asc&max_docs=$DOCS_DROP&refresh=true&q=*" >/dev/null + sleep 5 + log "reclaim deleted space" + curl -sS -XPOST "$ES_URL/*-$TODAY/_forcemerge?only_expunge_deletes=true" >/dev/null + sleep 30 + fi + log "flush" + curl -sS -XPOST "$ES_URL/*-$TODAY/_flush" >/dev/null + sleep 5 +done + +log "job exited" diff --git a/docker-build/elasticsearch/elasticsearch-setindex b/docker-build/elasticsearch/elasticsearch-setindex.sh old mode 100644 new mode 100755 similarity index 78% rename from docker-build/elasticsearch/elasticsearch-setindex rename to docker-build/elasticsearch/elasticsearch-setindex.sh index cb0fdcb..d931e3b --- a/docker-build/elasticsearch/elasticsearch-setindex +++ b/docker-build/elasticsearch/elasticsearch-setindex.sh @@ -11,13 +11,12 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License. ack="false" - while [ "$ack" != "true" ] do - response=$(curl -XPUT "localhost:9200/_template/default_template" -H 'Content-Type: application/json' -d' + response=$(curl -sS -XPUT "http://localhost:$ELASTICSEARCH_LOGGING_SERVICE_PORT/_template/default_template" -H 'Content-Type: application/json' -d' { "index_patterns": ["*"], "settings": { @@ -26,7 +25,7 @@ do } } }') - echo "number_of_replicas xput has been sent"; + echo "number_of_replicas=2 has been requested"; if [ "$response" = "{\"acknowledged\":true}" ]; then echo "number_of_replicas is set to 2"; ack="true"; fi sleep 1 done diff --git a/docker-build/elasticsearch/healthcheck b/docker-build/elasticsearch/healthcheck deleted file mode 100644 index 5d1c971..0000000 --- a/docker-build/elasticsearch/healthcheck +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Copyright 2019 Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -host="$(hostname --ip-address || echo '127.0.0.1')" - -if health="$(curl -fsSL "http://$host:${ELASTICSEARCH_PORT}/_cat/health?h=status")"; then - health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ") - if [ "$health" = 'green' ]; then - exit 0 - fi - echo >&2 "unexpected health status: $health" -fi - -# If the probe returns 2 ("starting") when the container has already moved out of the "starting" state then it is treated as "unhealthy" instead. -# https://github.com/docker/docker/blob/dcc65376bac8e73bb5930fce4cddc2350bb7baa2/docs/reference/builder.md#healthcheck -exit 2 diff --git a/docker-build/elasticsearch/config/elastic/log4j2.properties b/docker-build/elasticsearch/log4j2.properties similarity index 78% rename from docker-build/elasticsearch/config/elastic/log4j2.properties rename to docker-build/elasticsearch/log4j2.properties index 46877d0..9759246 100644 --- a/docker-build/elasticsearch/config/elastic/log4j2.properties +++ b/docker-build/elasticsearch/log4j2.properties @@ -1,9 +1,9 @@ status = error - appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - rootLogger.level = info rootLogger.appenderRef.console.ref = console +logger.searchguard.name = com.floragunn +logger.searchguard.level = info diff --git a/docker-build/fluentd/bom.json b/docker-build/fluentd/bom.json deleted file mode 100644 index a3f6f4d..0000000 --- a/docker-build/fluentd/bom.json +++ /dev/null @@ -1,622 +0,0 @@ -{ - "bom": [ - { - "name": "alpine-linux-container", - "version": "3.9", - "source-url": "https://github.com/gliderlabs/docker-alpine/archive/c4f4c7a6e14d6efeb9a160da464717e03d2cc3ee.tar.gz", - "foss": "yes" - }, - { - "name": "musl", - "version": "1.1.20-r3", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/musl/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "busybox", - "version": "1.29.3-r10", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/busybox/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "alpine-baselayout", - "version": "3.1.0-r3", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "alpine-keys", - "version": "2.1-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/alpine-keys/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "libcrypto1.1", - "version": "1.1.1a-r1", - "source-url": "https://git.alpinelinux.org/aports/tree/main/openssl/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "libssl1.1", - "version": "1.1.1a-r1", - "source-url": "https://git.alpinelinux.org/aports/tree/main/openssl/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ca-certificates-cacert", - "version": "20190108-r0", - "source-url": "https://git.alpinelinux.org/aports/tree/main/ca-certificates/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "libtls-standalone", - "version": "2.7.4-r6", - "source-url": "https://git.alpinelinux.org/aports/tree/main/libtls-standalone/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ssl_client", - "version": "1.29.3-r10", - "source-url": "https://git.alpinelinux.org/aports/tree/main/busybox/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "zlib", - "version": "1.2.11-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/zlib/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "apk-tools", - "version": "2.10.3-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/apk-tools/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "scanelf", - "version": "1.2.3-r0", - "source-url": "https://git.alpinelinux.org/aports/tree/main/pax-utils/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "musl-utils", - "version": "1.1.20-r3", - "source-url": "https://git.alpinelinux.org/aports/tree/main/musl/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "libc-utils", - "version": "0.7.1-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/libc-dev/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "libbz2", - "version": "1.0.6-r6", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/bzip2/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "expat", - "version": "2.2.6-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/expat/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "libffi", - "version": "3.2.1-r6", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/libffi/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "gdbm", - "version": "1.13-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/gdbm/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ncurses-terminfo-base", - "version": "6.1_p20190105-r0", - "source-url": "https://git.alpinelinux.org/aports/tree/main/ncurses/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ncurses-terminfo", - "version": "6.1_p20190105-r0", - "source-url": "https://git.alpinelinux.org/aports/tree/main/ncurses/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ncurses-libs", - "version": "6.1_p20190105-r0", - "source-url": "https://git.alpinelinux.org/aports/tree/main/ncurses/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "readline", - "version": "7.0.003-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/readline/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "sqlite-libs", - "version": "3.26.0-r3", - "source-url": "https://git.alpinelinux.org/aports/tree/main/sqlite/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "python2", - "version": "2.7.15-r3", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/python2/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "py-setuptools", - "version": "40.6.3-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/py-setuptools/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "py2-pip", - "version": "18.1-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/py2-pip/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ca-certificates", - "version": "20190108-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/ca-certificates/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "dumb-init", - "version": "1.2.2-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/dumb-init/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "gmp", - "version": "6.1.2-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/gmp/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "yaml", - "version": "0.2.1-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/yaml/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ruby-libs", - "version": "2.5.3-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/ruby-libs/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ruby", - "version": "2.5.3-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/ruby/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "ruby-irb", - "version": "2.5.3-r1", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/ruby-irb/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "su-exec", - "version": "0.2-r0", - "source-url": "https://git.alpinelinux.org/cgit/aports/tree/main/su-exec/APKBUILD?h=3.9-stable", - "foss": "yes" - }, - { - "name": "pip", - "version": "19.0.2", - "source-url": "https://github.com/pypa/pip/archive/19.0.2.tar.gz", - "foss": "yes" - }, - { - "name": "setuptools", - "version": "40.6.3.post20190116", - "source-url": "https://github.com/pypa/setuptools/archive/v40.6.3.tar.gz", - "foss": "yes" - }, - { - "name": "activesupport", - "version": "5.2.2", - "source-url": "https://github.com/rails/rails/archive/v5.2.2.tar.gz", - "foss": "yes" - }, - { - "name": "addressable", - "version": "2.6.0", - "source-url": "https://github.com/sporkmonger/addressable/archive/addressable-2.6.0.tar.gz", - "foss": "yes" - }, - { - "name": "bigdecimal", - "version": "1.4.3", - "source-url": "https://github.com/ruby/bigdecimal/archive/v1.4.3.tar.gz", - "foss": "yes" - }, - { - "name": "cmath", - "version": "1.0.0", - "source-url": "https://github.com/ruby/cmath/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "concurrent-ruby", - "version": "1.1.4", - "source-url": "https://github.com/ruby-concurrency/concurrent-ruby/archive/v1.1.4.tar.gz", - "foss": "yes" - }, - { - "name": "cool.io", - "version": "1.5.3", - "source-url": "https://github.com/tarcieri/cool.io/archive/v1.5.3.tar.gz", - "foss": "yes" - }, - { - "name": "csv", - "version": "1.0.0", - "source-url": "https://github.com/ruby/csv/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "date", - "version": "1.0.0", - "source-url": "https://github.com/ruby/date/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "dig_rb", - "version": "1.0.1", - "source-url": "https://github.com/jrochkind/dig_rb/archive/v1.0.1.tar.gz", - "foss": "yes" - }, - { - "name": "domain_name", - "version": "0.5.20180417", - "source-url": "https://github.com/knu/ruby-domain_name/archive/v0.5.20180417.tar.gz", - "foss": "yes" - }, - { - "name": "elasticsearch", - "version": "6.1.0", - "source-url": "https://github.com/elastic/elasticsearch-ruby/archive/v6.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "elasticsearch-api", - "version": "6.1.0", - "source-url": "https://github.com/elastic/elasticsearch-ruby/archive/v6.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "elasticsearch-transport", - "version": "6.1.0", - "source-url": "https://github.com/elastic/elasticsearch-ruby/archive/v6.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "etc", - "version": "1.0.1", - "source-url": "https://github.com/ruby/etc/archive/v1.0.1.tar.gz", - "foss": "yes" - }, - { - "name": "excon", - "version": "0.62.0", - "source-url": "https://github.com/excon/excon/archive/v0.62.0.tar.gz", - "foss": "yes" - }, - { - "name": "faraday", - "version": "0.15.4", - "source-url": "https://github.com/lostisland/faraday/archive/v0.15.4.tar.gz", - "foss": "yes" - }, - { - "name": "fcntl", - "version": "1.0.0", - "source-url": "https://github.com/ruby/fcntl/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "fileutils", - "version": "1.0.2", - "source-url": "https://github.com/ruby/fileutils/archive/v1.0.2.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-config-regexp-type", - "version": "1.0.0", - "source-url": "https://github.com/okkez/fluent-config-regexp-type/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-plugin-elasticsearch", - "version": "3.2.1", - "source-url": "https://github.com/uken/fluent-plugin-elasticsearch/archive/v3.2.1.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-plugin-remote_syslog", - "version": "1.0.0", - "source-url": "https://github.com/dlackty/fluent-plugin-remote_syslog/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "remote_syslog_sender", - "version": "1.2.1", - "source-url": "https://github.com/reproio/remote_syslog_sender/archive/v1.2.1.tar.gz", - "foss": "yes" - }, - { - "name": "syslog_protocol", - "version": "0.9.2", - "source-url": "https://github.com/eric/syslog_protocol/archive/v0.9.2.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-plugin-kubernetes_metadata_filter", - "version": "2.1.6", - "source-url": "https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/archive/v2.1.6.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-plugin-record-reformer", - "version": "0.9.1", - "source-url": "https://github.com/sonots/fluent-plugin-record-reformer/archive/v0.9.1.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-plugin-rewrite-tag-filter", - "version": "2.1.1", - "source-url": "https://github.com/fluent/fluent-plugin-rewrite-tag-filter/archive/v2.1.1.tar.gz", - "foss": "yes" - }, - { - "name": "fluent-plugin-secure-forward", - "version": "0.4.5", - "source-url": "https://github.com/tagomoris/fluent-plugin-secure-forward/archive/v0.4.5.tar.gz", - "foss": "yes" - }, - { - "name": "fluentd", - "version": "1.3.3", - "source-url": "https://github.com/fluent/fluentd/archive/v1.3.3.tar.gz", - "foss": "yes" - }, - { - "name": "http", - "version": "0.9.8", - "source-url": "https://github.com/httprb/http/archive/v0.9.8.tar.gz", - "foss": "yes" - }, - { - "name": "http-cookie", - "version": "1.0.3", - "source-url": "https://github.com/sparklemotion/http-cookie/archive/v1.0.3.tar.gz", - "foss": "yes" - }, - { - "name": "http-form_data", - "version": "1.0.3", - "source-url": "https://github.com/httprb/form_data/archive/v1.0.3.tar.gz", - "foss": "yes" - }, - { - "name": "http_parser.rb", - "version": "0.6.0", - "source-url": "https://github.com/tmm1/http_parser.rb/archive/v0.6.0.tar.gz", - "foss": "yes" - }, - { - "name": "i18n", - "version": "1.5.3", - "source-url": "https://github.com/ruby-i18n/i18n/archive/v1.5.3.tar.gz", - "foss": "yes" - }, - { - "name": "ipaddr", - "version": "1.2.0", - "source-url": "https://github.com/ruby/ipaddr/archive/v1.2.0.tar.gz", - "foss": "yes" - }, - { - "name": "json", - "version": "2.1.0", - "source-url": "https://github.com/flori/json/archive/v2.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "kubeclient", - "version": "1.1.4", - "source-url": "https://github.com/abonas/kubeclient/archive/v1.1.4.tar.gz", - "foss": "yes" - }, - { - "name": "lru_redux", - "version": "1.1.0", - "source-url": "https://github.com/SamSaffron/lru_redux/archive/v1.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "mime-types", - "version": "3.2.2", - "source-url": "https://github.com/mime-types/ruby-mime-types/archive/v3.2.2.tar.gz", - "foss": "yes" - }, - { - "name": "mime-types-data", - "version": "3.2018.0812", - "source-url": "https://github.com/mime-types/mime-types-data/archive/v3.2018.0812.tar.gz", - "foss": "yes" - }, - { - "name": "minitest", - "version": "5.11.3", - "source-url": "https://github.com/seattlerb/minitest/archive/f69797251ef5499250ba4652a7df0b941ef90641.tar.gz", - "foss": "yes" - }, - { - "name": "msgpack", - "version": "1.2.6", - "source-url": "https://github.com/msgpack/msgpack-ruby/archive/v1.2.6.tar.gz", - "foss": "yes" - }, - { - "name": "multi_json", - "version": "1.13.1", - "source-url": "https://github.com/intridea/multi_json/archive/v1.13.1.tar.gz", - "foss": "yes" - }, - { - "name": "multipart-post", - "version": "2.0.0", - "source-url": "https://github.com/nicksieger/multipart-post/archive/v2.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "netrc", - "version": "0.11.0", - "source-url": "https://github.com/heroku/netrc/archive/6ce9be04800aae0df1b7d44ce195117485de8b6a.tar.gz", - "foss": "yes" - }, - { - "name": "oj", - "version": "2.18.3", - "source-url": "https://github.com/ohler55/oj/archive/v2.18.3.tar.gz", - "foss": "yes" - }, - { - "name": "openssl", - "version": "2.1.2", - "source-url": "https://github.com/ruby/openssl/archive/v2.1.2.tar.gz", - "foss": "yes" - }, - { - "name": "proxifier", - "version": "1.0.3", - "source-url": "https://github.com/samuelkadolph/ruby-proxifier/archive/v1.0.3.tar.gz", - "foss": "yes" - }, - { - "name": "psych", - "version": "3.0.2", - "source-url": "https://github.com/ruby/psych/archive/v3.0.2.tar.gz", - "foss": "yes" - }, - { - "name": "public_suffix", - "version": "3.0.3", - "source-url": "https://github.com/weppos/publicsuffix-ruby/archive/v3.0.3.tar.gz", - "foss": "yes" - }, - { - "name": "recursive-open-struct", - "version": "1.0.0", - "source-url": "https://github.com/aetherknight/recursive-open-struct/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "resolve-hostname", - "version": "0.1.0", - "source-url": "https://github.com/tagomoris/resolve-hostname/archive/v0.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "rest-client", - "version": "2.0.2", - "source-url": "https://github.com/rest-client/rest-client/archive/v2.0.2.tar.gz", - "foss": "yes" - }, - { - "name": "scanf", - "version": "1.0.0", - "source-url": "https://github.com/ruby/scanf/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "serverengine", - "version": "2.1.0", - "source-url": "https://github.com/treasure-data/serverengine/archive/v2.1.0.tar.gz", - "foss": "yes" - }, - { - "name": "sigdump", - "version": "0.2.4", - "source-url": "https://github.com/frsyuki/sigdump/archive/v0.2.4.tar.gz", - "foss": "yes" - }, - { - "name": "stringio", - "version": "0.0.1", - "source-url": "https://github.com/ruby/stringio/archive/4c3b508e686b387f244ffd2e7d66af4d9db6f752.tar.gz", - "foss": "yes" - }, - { - "name": "strptime", - "version": "0.2.3", - "source-url": "https://github.com/nurse/strptime/archive/v0.2.3.tar.gz", - "foss": "yes" - }, - { - "name": "strscan", - "version": "1.0.0", - "source-url": "https://github.com/ruby/strscan/archive/v1.0.0.tar.gz", - "foss": "yes" - }, - { - "name": "thread_safe", - "version": "0.3.6", - "source-url": "https://github.com/ruby-concurrency/thread_safe/archive/v0.3.6.tar.gz", - "foss": "yes" - }, - { - "name": "tzinfo", - "version": "1.2.5", - "source-url": "https://github.com/tzinfo/tzinfo/archive/v1.2.5.tar.gz", - "foss": "yes" - }, - { - "name": "tzinfo-data", - "version": "1.2018.9", - "source-url": "https://github.com/tzinfo/tzinfo-data/archive/v1.2018.9.tar.gz", - "foss": "yes" - }, - { - "name": "unf", - "version": "0.1.4", - "source-url": "https://github.com/knu/ruby-unf/archive/v0.1.4.tar.gz", - "foss": "yes" - }, - { - "name": "unf_ext", - "version": "0.0.7.5", - "source-url": "https://github.com/knu/ruby-unf_ext/archive/v0.0.7.5.tar.gz", - "foss": "yes" - }, - { - "name": "webrick", - "version": "1.4.2", - "source-url": "https://svn.ruby-lang.org/repos/ruby/tags/v1_4_2/", - "foss": "yes" - }, - { - "name": "yajl-ruby", - "version": "1.4.1", - "source-url": "https://github.com/brianmario/yajl-ruby/archive/1.4.1.tar.gz", - "foss": "yes" - }, - { - "name": "zlib", - "version": "1.0.0", - "source-url": "https://github.com/ruby/zlib/archive/v1.0.0.tar.gz", - "foss": "yes" - } - ] -}