From 30303fddd4b067b52aad6d45b5718097de3ee43a Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Thu, 14 Mar 2019 11:54:30 +0000
Subject: Add Elasticsearch Curator Docker image

Kolla provides log aggregation, but no solution to set retention periods
for aggregated logs. It therefore accumulates log data indefinitely unless
a user manually intervenes. This change adds Elasticsearch Curator which
provides a mechanism for automating such retention periods [1].

[1] https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html

The container contains cron to support running Curator periodically.

Change-Id: Ief2c554a64ef6cc971635d7e2a718f63c310fbf6
(cherry picked from commit 37de8920aa7394089731c0fae75034c2c3d137ca)
---
 docker/elasticsearch/Dockerfile.j2                 | 43 ----------------------
 .../elasticsearch-curator/Dockerfile.j2            | 40 ++++++++++++++++++++
 .../elasticsearch-curator/extend_start.sh          |  8 ++++
 docker/elasticsearch/elasticsearch/Dockerfile.j2   | 43 ++++++++++++++++++++++
 docker/elasticsearch/elasticsearch/extend_start.sh |  8 ++++
 docker/elasticsearch/extend_start.sh               |  8 ----
 ...add-elasticsearch-curator-88089d04f7ccd549.yaml |  4 ++
 7 files changed, 103 insertions(+), 51 deletions(-)
 delete mode 100644 docker/elasticsearch/Dockerfile.j2
 create mode 100644 docker/elasticsearch/elasticsearch-curator/Dockerfile.j2
 create mode 100644 docker/elasticsearch/elasticsearch-curator/extend_start.sh
 create mode 100644 docker/elasticsearch/elasticsearch/Dockerfile.j2
 create mode 100644 docker/elasticsearch/elasticsearch/extend_start.sh
 delete mode 100644 docker/elasticsearch/extend_start.sh
 create mode 100644 releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml

diff --git a/docker/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/Dockerfile.j2
deleted file mode 100644
index 7856444b9..000000000
--- a/docker/elasticsearch/Dockerfile.j2
+++ /dev/null
@@ -1,43 +0,0 @@
-FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
-
-{% block elasticsearch_header %}{% endblock %}
-
-{% import "macros.j2" as macros with context %}
-
-{{ macros.configure_user(name='elasticsearch', shell='/bin/bash', homedir='/usr/share/elasticsearch') }}
-
-{% if base_package_type == 'rpm' %}
-    {% set elasticsearch_packages = [
-        'elasticsearch',
-        'java-1.8.0-openjdk-headless',
-    ] %}
-
-ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
-
-{% elif base_package_type == 'deb' %}
-    {% set elasticsearch_packages = [
-        'default-jre-headless',
-        'elasticsearch',
-    ] %}
-
-ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
-
-{% endif %}
-
-ENV ES_SKIP_SET_KERNEL_PARAMETERS true
-
-{{ macros.install_packages(elasticsearch_packages | customizable("packages")) }}
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-
-# NOTE: By default the shell of the elasticsearch user is /bin/false. We have to
-# change the shell to /bin/bash to be able to run elasticsearch as a non-root user.
-#
-# https://discuss.elastic.co/t/running-as-non-root-user-service-wrapper-has-changed/7863
-
-RUN chmod 755 /usr/local/bin/kolla_extend_start
-
-{% block elasticsearch_footer %}{% endblock %}
-{% block footer %}{% endblock %}
-
-USER elasticsearch
diff --git a/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2 b/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2
new file mode 100644
index 000000000..7497163bc
--- /dev/null
+++ b/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2
@@ -0,0 +1,40 @@
+FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+
+{% block elasticsearch_curator_header %}{% endblock %}
+
+{% import "macros.j2" as macros with context %}
+
+{% if base_package_type == 'rpm' %}
+    {% set elasticsearch_curator_packages = [
+        'cronie',
+        'python3-pip',
+    ] %}
+{% elif base_package_type == 'deb' %}
+    {% set elasticsearch_curator_packages = [
+        'cron',
+        'python3-pip',
+        'python3-setuptools'
+    ] %}
+{% endif %}
+
+{{ macros.install_packages(elasticsearch_curator_packages | customizable("packages")) }}
+
+{{ macros.configure_user(name='elasticsearch') }}
+
+{% set elasticsearch_curator_pip_packages = [
+    'elasticsearch-curator'
+] %}
+
+RUN {{ macros.install_pip(elasticsearch_curator_pip_packages | customizable("pip_packages"), constraints=false, pip_version="pip3") }} \
+    && mkdir -p /etc/elasticsearch-curator \
+    && chown -R elasticsearch: /etc/elasticsearch-curator
+
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start
+
+{% block elasticsearch_curator_base_footer %}{% endblock %}
+{% block footer %}{% endblock %}
+
+# NOTE(dszumski): Cron runs as root but should be configured to launch Curator
+# by the elasticsearch user.
diff --git a/docker/elasticsearch/elasticsearch-curator/extend_start.sh b/docker/elasticsearch/elasticsearch-curator/extend_start.sh
new file mode 100644
index 000000000..8af0cdd15
--- /dev/null
+++ b/docker/elasticsearch/elasticsearch-curator/extend_start.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# NOTE(dszumski): Cron runs as root but should be configured to launch Curator
+# by the elasticsearch user. Therefore this directory needs to be owned by
+# the elasticsearch user.
+if [[ ! -d "/var/log/kolla/elasticsearch" ]]; then
+    install -d -m 0755 -o elasticsearch -g elasticsearch /var/log/kolla/elasticsearch
+fi
diff --git a/docker/elasticsearch/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/elasticsearch/Dockerfile.j2
new file mode 100644
index 000000000..7856444b9
--- /dev/null
+++ b/docker/elasticsearch/elasticsearch/Dockerfile.j2
@@ -0,0 +1,43 @@
+FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+
+{% block elasticsearch_header %}{% endblock %}
+
+{% import "macros.j2" as macros with context %}
+
+{{ macros.configure_user(name='elasticsearch', shell='/bin/bash', homedir='/usr/share/elasticsearch') }}
+
+{% if base_package_type == 'rpm' %}
+    {% set elasticsearch_packages = [
+        'elasticsearch',
+        'java-1.8.0-openjdk-headless',
+    ] %}
+
+ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
+
+{% elif base_package_type == 'deb' %}
+    {% set elasticsearch_packages = [
+        'default-jre-headless',
+        'elasticsearch',
+    ] %}
+
+ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
+
+{% endif %}
+
+ENV ES_SKIP_SET_KERNEL_PARAMETERS true
+
+{{ macros.install_packages(elasticsearch_packages | customizable("packages")) }}
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+
+# NOTE: By default the shell of the elasticsearch user is /bin/false. We have to
+# change the shell to /bin/bash to be able to run elasticsearch as a non-root user.
+#
+# https://discuss.elastic.co/t/running-as-non-root-user-service-wrapper-has-changed/7863
+
+RUN chmod 755 /usr/local/bin/kolla_extend_start
+
+{% block elasticsearch_footer %}{% endblock %}
+{% block footer %}{% endblock %}
+
+USER elasticsearch
diff --git a/docker/elasticsearch/elasticsearch/extend_start.sh b/docker/elasticsearch/elasticsearch/extend_start.sh
new file mode 100644
index 000000000..b39f5b334
--- /dev/null
+++ b/docker/elasticsearch/elasticsearch/extend_start.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ ! -d "/var/log/kolla/elasticsearch" ]]; then
+    mkdir -p /var/log/kolla/elasticsearch
+fi
+if [[ $(stat -c %a /var/log/kolla/elasticsearch) != "755" ]]; then
+    chmod 755 /var/log/kolla/elasticsearch
+fi
diff --git a/docker/elasticsearch/extend_start.sh b/docker/elasticsearch/extend_start.sh
deleted file mode 100644
index b39f5b334..000000000
--- a/docker/elasticsearch/extend_start.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-if [[ ! -d "/var/log/kolla/elasticsearch" ]]; then
-    mkdir -p /var/log/kolla/elasticsearch
-fi
-if [[ $(stat -c %a /var/log/kolla/elasticsearch) != "755" ]]; then
-    chmod 755 /var/log/kolla/elasticsearch
-fi
diff --git a/releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml b/releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml
new file mode 100644
index 000000000..79bd070ca
--- /dev/null
+++ b/releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds Elasticsearch Curator for managing aggregated log data.
-- 
cgit v1.2.3