Upgrade to Logstash 6

Elasticsearch and Kibana 6 are used on all distros, but Logstash
is still pinned at 2 which is not compatible. See the product
compatibility matrix [1].

[1] https://www.elastic.co/support/matrix#matrix_compatibility

Partial-Bug: #1884090
Change-Id: I1579e762bcfb07f0baa73507eb9955ae2f83c4ec

4 files changed, 25 insertions, 49 deletions

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index fd2ced890..f761da6dd 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -104,14 +104,11 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
     ] %}
 {% elif base_arch == 'aarch64' %}
     {% set base_yum_repo_files = [
+        'elasticsearch.repo',
         'grafana.repo',
         'rabbitmq_rabbitmq-server.repo',
         'rabbitmq_rabbitmq-erlang-aarch64.repo',
     ] %}
-    # FIXME(mgoddard): Not available for CentOS 8 yet.
-    #
-    #   'elasticsearch.repo',
-    #
 
     {% set base_yum_repo_keys = [
         'https://packages.grafana.com/gpg.key',
diff --git a/docker/logstash/Dockerfile.j2 b/docker/logstash/Dockerfile.j2
index 3e8bcdb1f..57a5a8460 100644
--- a/docker/logstash/Dockerfile.j2
+++ b/docker/logstash/Dockerfile.j2
@@ -9,58 +9,26 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.configure_user(name='logstash', shell='/bin/bash', homedir='/usr/share/logstash') }}
 
-{% if base_package_type == 'rpm' %}
-    {% set logstash_packages = [
-        'java-1.8.0-openjdk-headless'
-    ] %}
-
-ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
+{{ macros.enable_extra_repos(['logstash']) }}
 
+{% if base_package_type == 'rpm' %}
+    {% set elk_java_package = 'java-11-openjdk-headless' %}
+ENV JAVA_HOME /usr/lib/jvm/jre-11-openjdk/
 {% elif base_package_type == 'deb' %}
-
-    {% if base_distro == 'debian' %}
-        {% set java_version = '11' %}
-    {% elif base_distro == 'ubuntu' %}
-        {% set java_version = '8' %}
-    {% endif %}
-
-    {% set logstash_packages = [
-        'openjdk-' + java_version + '-jre-headless',
-        'logrotate'
-    ] %}
-
-ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
-
+    {% set elk_java_package = 'openjdk-11-jre-headless' %}
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-{{debian_arch}}/
 {% endif %}
 
+{% set logstash_packages = [
+    'logstash-oss',
+] %}
+
+# Post-install script of Logstash fails when trying to install Logstash and
+# Java together.
+{{ macros.install_packages([elk_java_package]) }}
 {{ macros.install_packages(logstash_packages | customizable("packages")) }}
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
 
-# FIXME(dszumski): Logstash was added to Kolla for Monasca. Monasca is
-# currently tied to an old messaging version in Kafka (0.9) and doesn't
-# work with Logstash 5.x or above. When Monasca is updated to use a more
-# recent messaging version this patch should be reverted and the Monasca
-# Logstash configuration files upgraded to use the new format.
-{% if base_package_type == 'rpm' %}
-
-ENV logstash_rpm_url=https://download.elastic.co/logstash/logstash/packages/centos/logstash-2.4.1.noarch.rpm
-ENV logstash_rpm_sha1sum=68c5e78c3d941f06f77d638c0756b6aea715bec5
-RUN curl -sSL -o /tmp/logstash.rpm ${logstash_rpm_url} \
-    && echo "${logstash_rpm_sha1sum} /tmp/logstash.rpm" | sha1sum -c \
-    && dnf -y install /tmp/logstash.rpm \
-    && rm -f /tmp/logstash.rpm
-
-{% elif base_package_type == 'deb' %}
-
-ENV logstash_deb_url=https://download.elastic.co/logstash/logstash/packages/debian/logstash-2.4.1_all.deb
-ENV logstash_deb_sha1sum=7ba3b174a3ef48a7d0945d9b5c7f12c5005abb47
-RUN curl -sSL -o /tmp/logstash.deb ${logstash_deb_url} \
-    && echo "${logstash_deb_sha1sum} /tmp/logstash.deb" | sha1sum -c \
-    && apt install /tmp/logstash.deb \
-    && rm -f /tmp/logstash.deb
-
-{% endif %}
-
 RUN chmod 755 /usr/local/bin/kolla_extend_start
 
 {% block logstash_footer %}{% endblock %}
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index 29434a529..52fb7565d 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -10,6 +10,7 @@ centos:
   grafana: "grafana"
   hacluster: "HighAvailability"
   influxdb: "influxdb"
+  logstash: "elasticsearch-kibana-logstash-6.x"
   kibana: "elasticsearch-kibana-logstash-6.x"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
@@ -24,6 +25,7 @@ centos-aarch64:
   extras: "extras"
   grafana: "grafana"
   hacluster: "HighAvailability"
+  logstash: "elasticsearch-kibana-logstash-6.x"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
 
@@ -34,6 +36,7 @@ centos-ppc64le:
   epel-modular: "epel-modular"
   extras: "extras"
   hacluster: "HighAvailability"
+  logstash: "elasticsearch-kibana-logstash-6.x"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
 
@@ -48,6 +51,7 @@ debian:
   erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
+  logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
   td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib"
@@ -57,6 +61,7 @@ debian-aarch64:
   erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
+  logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
@@ -66,6 +71,7 @@ ubuntu:
   erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
+  logstash: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   kibana: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   opendaylight: "deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main"
@@ -79,6 +85,7 @@ ubuntu-aarch64:
   erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
+  logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"
 
diff --git a/releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml b/releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml
new file mode 100644
index 000000000..e0b6a28d0
--- /dev/null
+++ b/releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml
@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    The Logstash image has been upgraded from Logstash 2 to Logstash 6.