Upgrade RabbitMQ to 3.8

Upgrade and pin RabbitMQ to 3.8.

Upgrade Erlang to 22 to meet requirements:
https://www.rabbitmq.com/which-erlang.html

Start using bintray repository provided by RabbitMQ.

Linaro builds will be used for aarch64.

Implements: blueprint rabbitmq-version-upgrade

Closes-Bug: #1848452

Change-Id: I3d775fd832f2603181da89009cebc1370a61a023

11 files changed, 67 insertions, 55 deletions

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index 06e7b3693..5bba199ed 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -93,6 +93,7 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
         'grafana.repo',
         'influxdb.repo',
         'rabbitmq_rabbitmq-server.repo',
+        'rabbitmq_rabbitmq-erlang.repo',
         'td.repo',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
@@ -107,6 +108,7 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
     {% set base_yum_repo_keys = [
         'https://packages.grafana.com/gpg.key',
         'https://repos.influxdata.com/influxdb.key',
+        'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
         'https://packages.treasuredata.com/GPG-KEY-td-agent',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
@@ -119,6 +121,7 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
     {% set base_yum_repo_files = [
         'grafana.repo',
         'rabbitmq_rabbitmq-server.repo',
+        'rabbitmq_rabbitmq-erlang-aarch64.repo',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
     {% if distro_package_manager == 'yum' %}
@@ -129,6 +132,8 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
 
     {% set base_yum_repo_keys = [
         'https://packages.grafana.com/gpg.key',
+        'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
+        'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/centos7/repodata/repomd.xml.key',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
     {% if distro_package_manager == 'yum' %}
@@ -437,10 +442,15 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       'F1656F24C74CD1D8',
     ] %}
     {% set remote_apt_keys = [
-      'https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey',
+      'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
       'https://packages.grafana.com/gpg.key',
-      'https://repos.influxdata.com/influxdb.key'
+      'https://repos.influxdata.com/influxdb.key',
     ] %}
+    {% if base_arch == 'aarch64' %}
+        {% set remote_apt_keys = remote_apt_keys + [
+          'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic/Release.key',
+        ] %}
+    {% endif %}
 {% elif base_distro == 'debian' %}
     {% set base_apt_keys = [
       '46095ACC8548582C1A2699A9D27D666CD88E42B4',
@@ -452,7 +462,13 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       'https://packages.treasuredata.com/GPG-KEY-td-agent',
       'https://repos.influxdata.com/influxdb.key',
       'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key',
+      'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
     ] %}
+    {% if base_arch == 'aarch64' %}
+        {% set remote_apt_keys = remote_apt_keys + [
+          'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key',
+        ] %}
+    {% endif %}
     {% set base_apt_packages = base_apt_packages +
       ['sudo',]
     %}
diff --git a/docker/base/apt_preferences.debian b/docker/base/apt_preferences.debian
index 2b73f1c3e..6cd3b7719 100644
--- a/docker/base/apt_preferences.debian
+++ b/docker/base/apt_preferences.debian
@@ -8,6 +8,11 @@ Package: smartmontools
 Pin: version 7.*
 Pin-Priority: 1000
 
+# We do not pin minor versions anymore
+Package: rabbitmq-server
+Pin: version 3.8.*
+Pin-Priority: 1000
+
 # We do not want packages from this repo
 Package: *
 Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
diff --git a/docker/base/apt_preferences.ubuntu b/docker/base/apt_preferences.ubuntu
index 8ee23861f..db1c1d4ed 100644
--- a/docker/base/apt_preferences.ubuntu
+++ b/docker/base/apt_preferences.ubuntu
@@ -1,3 +1,4 @@
+# We do not pin minor versions anymore
 Package: rabbitmq-server
-Pin: version 3.7.10-1
+Pin: version 3.8.*
 Pin-Priority: 1000
diff --git a/docker/base/rabbitmq_rabbitmq-erlang-aarch64.repo b/docker/base/rabbitmq_rabbitmq-erlang-aarch64.repo
new file mode 100644
index 000000000..a084c74c2
--- /dev/null
+++ b/docker/base/rabbitmq_rabbitmq-erlang-aarch64.repo
@@ -0,0 +1,7 @@
+[rabbitmq_rabbitmq-erlang]
+name=rabbitmq_rabbitmq-erlang
+baseurl=http://obs.linaro.org/home:/marcin.juszkiewicz/centos$releasever/
+repo_gpgcheck=0
+gpgcheck=1
+enabled=0
+gpgkey=http://obs.linaro.org/home:/marcin.juszkiewicz/centos$releasever/repodata/repomd.xml.key
diff --git a/docker/base/rabbitmq_rabbitmq-erlang.repo b/docker/base/rabbitmq_rabbitmq-erlang.repo
new file mode 100644
index 000000000..a3ecea85c
--- /dev/null
+++ b/docker/base/rabbitmq_rabbitmq-erlang.repo
@@ -0,0 +1,7 @@
+[rabbitmq_rabbitmq-erlang]
+name=rabbitmq_rabbitmq-erlang
+baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/22/el/$releasever/
+repo_gpgcheck=0
+gpgcheck=1
+enabled=0
+gpgkey=https://www.rabbitmq.com/rabbitmq-release-signing-key.asc
diff --git a/docker/base/rabbitmq_rabbitmq-server.repo b/docker/base/rabbitmq_rabbitmq-server.repo
index 7bc839615..2a04f4521 100644
--- a/docker/base/rabbitmq_rabbitmq-server.repo
+++ b/docker/base/rabbitmq_rabbitmq-server.repo
@@ -1,13 +1,11 @@
 [rabbitmq_rabbitmq-server]
 name=rabbitmq_rabbitmq-server
-baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/$releasever/$basearch
+baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/$releasever/
 # NOTE(mgoddard): rabbitmq-server 3.7 packages are not signed. Previously
 # repo_gpgcheck was set to 1, but this breaks DNF which does not seem to accept
 # keys imported by rpm --import for signing a repo (as opposed to packages).
 # TODO(mgoddard): Set gpgcheck=1 for rabbitmq-server 3.8+
-gpgcheck=0
+repo_gpgcheck=0
+gpgcheck=1
 enabled=0
-gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
-sslverify=1
-sslcacert=/etc/pki/tls/certs/ca-bundle.crt
-metadata_expire=300
+gpgkey=https://www.rabbitmq.com/rabbitmq-release-signing-key.asc
diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian
index 0f88b7cdb..109ec3e1f 100644
--- a/docker/base/sources.list.debian
+++ b/docker/base/sources.list.debian
@@ -25,3 +25,12 @@ deb http://buster-train.debian.net/debian buster-train-backports-nochange main
 
 # td-agent for fluentd
 #deb http://packages.treasuredata.com/3/debian/buster buster contrib
+
+# rabbitmq repo
+#deb https://dl.bintray.com/rabbitmq/debian/ buster main
+
+# erlang repo
+#deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang
+
+# erlang repo - aarch64 only
+#deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./
diff --git a/docker/base/sources.list.ubuntu b/docker/base/sources.list.ubuntu
index 5bfc3a10e..9a9fbf72b 100644
--- a/docker/base/sources.list.ubuntu
+++ b/docker/base/sources.list.ubuntu
@@ -40,7 +40,13 @@ deb http://ubuntu-cloud.archive.canonical.com/ubuntu bionic-updates/ussuri main
 #deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main
 
 # rabbitmq repo
-#deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
+#deb https://dl.bintray.com/rabbitmq/debian/ bionic main
+
+# erlang repo
+#deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang
+
+# erlang repo - aarch64 only
+#deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./
 
 # MariaDB repo
 #deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main
diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2
index 44ebaf329..039d02e1c 100644
--- a/docker/kolla-toolbox/Dockerfile.j2
+++ b/docker/kolla-toolbox/Dockerfile.j2
@@ -13,10 +13,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {{ macros.configure_user(name='ansible') }}
 {{ macros.configure_user(name='rabbitmq') }}
 
-{{ macros.enable_extra_repos(['rabbitmq']) }}
+{{ macros.enable_extra_repos(['rabbitmq', 'erlang']) }}
 
-# NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-# https://bugs.launchpad.net/kolla/+bug/1814233
 {% if base_package_type == 'rpm' %}
     {% set kolla_toolbox_packages = [
         'crudini',
@@ -36,14 +34,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% if distro_package_manager == 'dnf' %}
         {% set kolla_toolbox_packages = kolla_toolbox_packages + [
             'python3-devel',
-            'rabbitmq-server-3.7.*'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% else %}
-        # NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-        # https://bugs.launchpad.net/kolla/+bug/1814233
         {% set kolla_toolbox_packages = kolla_toolbox_packages + [
             'python-devel',
-            'rabbitmq-server-3.7.10'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% endif %}
 
diff --git a/docker/macros.j2 b/docker/macros.j2
index ae6d683bc..bdccacfc4 100644
--- a/docker/macros.j2
+++ b/docker/macros.j2
@@ -87,15 +87,6 @@ RUN usermod --append --home {{ homedir }} --groups kolla {{ name }} \
 {%- endif %}
 {%- endmacro %}
 
-{% macro install_rabbitmq_plugins(plugins, version) -%}
-    ENV PLUGINS_DIR /usr/lib/rabbitmq/plugins
-{% if plugins is defined and plugins|length > 0 -%}
-    {% for item in plugins %}
-         RUN curl -L -o ${PLUGINS_DIR}/{{ item }} https://github.com/deadtrickster/prometheus_rabbitmq_exporter/releases/download/{{ version }}/{{ item }}
-    {% endfor %}
-{%- endif %}
-{%- endmacro %}
-
 {% macro enable_extra_repos(repos) %}
 
 {{ handle_repos(repos, 'enable') }}
diff --git a/docker/rabbitmq/Dockerfile.j2 b/docker/rabbitmq/Dockerfile.j2
index 046cb9b3c..385255749 100644
--- a/docker/rabbitmq/Dockerfile.j2
+++ b/docker/rabbitmq/Dockerfile.j2
@@ -9,44 +9,29 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.configure_user(name='rabbitmq') }}
 
-{{ macros.enable_extra_repos(['rabbitmq']) }}
+{{ macros.enable_extra_repos(['rabbitmq', 'erlang']) }}
 
 {% block rabbitmq_install %}
 {% if base_package_type == 'rpm' %}
     {% set rabbitmq_packages = [
-        'erlang-hipe',
         'hostname',
     ] %}
     {% if distro_package_manager == 'dnf' %}
         {% set rabbitmq_packages = rabbitmq_packages + [
-            'rabbitmq-server-3.7.*'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% else %}
-        # NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-        # https://bugs.launchpad.net/kolla/+bug/1814233
         {% set rabbitmq_packages = rabbitmq_packages + [
-            'rabbitmq-server-3.7.10'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% endif %}
 
 {% elif base_package_type == 'deb' %}
     {% set rabbitmq_packages = [
-        'erlang-nox',
         'logrotate',
         'rabbitmq-server',
     ] %}
 
-    {% if base_arch == 'aarch64' %}
-         {% set rabbitmq_packages = rabbitmq_packages + [
-             'erlang-base',
-         ] %}
-    {% else %}
-         {% set rabbitmq_packages = rabbitmq_packages + [
-             'erlang-base-hipe',
-         ] %}
-
-    {% endif %}
-
 {% endif %}
 
 {{ macros.install_packages(rabbitmq_packages | customizable("packages")) }}
@@ -58,18 +43,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 RUN rm -rf /var/lib/rabbitmq/* \
     && mkdir -p /usr/lib/rabbitmq/plugins \
     && /usr/lib/rabbitmq/bin/rabbitmq-plugins enable --offline \
-       rabbitmq_management
-
-
-    {% set rabbitmq_plugins = [
-        'accept-0.3.3.ez',
-        'prometheus-3.5.1.ez',
-        'prometheus_cowboy-0.1.4.ez',
-        'prometheus_httpd-2.1.8.ez',
-        'prometheus_rabbitmq_exporter-3.7.2.4.ez'
-    ] %}
+       rabbitmq_management \
+       rabbitmq_prometheus
 
-{{ macros.install_rabbitmq_plugins(rabbitmq_plugins | customizable("plugins"), version="v3.7.2.4") }}
 {% endblock %}
 
 COPY extend_start.sh /usr/local/bin/kolla_extend_start