CentOS 8: base and openstack-base images

Adds support to the base and openstack-base images for CentOS 8.
These images have been removed from the list of unbuildable images, and
replaced with all dependent images.

This change also includes backports of a number of other changes related
to the base image:

* CentOS 8: disable repo_gpgcheck for grafana image
  (Ice120e8001fd9e8bb37ca4f2df26259cf798ee0e)
* CentOS 8: base image fixes for RabbitMQ, fluentd & InfluxDB
  (Ibbb720390ff17e11249a5aa77163c15c0971209a)

Change-Id: I18fba8a5c2f99b700990cbf41c76077a1ec2408a
Partially-Implements: blueprint centos-rhel-8
(cherry picked from commit 3c618a1f6f0beca723e031fbe323b159400ad08f)

9 files changed, 263 insertions, 41 deletions

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index 6a7ea3f33..357f1da1f 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -42,9 +42,11 @@ ENV PS1="$(tput bold)($(printenv KOLLA_SERVICE_NAME))$(tput sgr0)[$(id -un)@$(ho
 # enables to provide repo overrides at a later date in a simple fashion if we
 # desire such functionality.  I think we will :)
 
+{# FIXME(mgoddard): Remove special case for CentOS 8 when CentOS 7 is no #}
+{# longer supported. #}
 RUN CURRENT_DISTRO_RELEASE=$(awk '{match($0, /[0-9]+/,version)}END{print version[0]}' /etc/system-release); \
-    if [  $CURRENT_DISTRO_RELEASE != "{{ supported_distro_release }}" ]; then \
-        echo "Only release '{{ supported_distro_release }}' is supported on {{ base_distro }}"; false; \
+    if [  $CURRENT_DISTRO_RELEASE != "{{ supported_distro_release }}" ] && [ $CURRENT_DISTRO_RELEASE != 8 ]; then \
+        echo "Only releases '{{ supported_distro_release }}' and '8' are supported on {{ base_distro }}"; false; \
     fi \
     && cat /tmp/kolla_bashrc >> /etc/bashrc \
     && sed -i 's|^\(override_install_langs=.*\)|# \1|' {% if distro_package_manager == 'dnf' %}/etc/dnf/dnf.conf{% else %}/etc/yum.conf{% endif %}
@@ -59,8 +61,22 @@ COPY yum.conf /etc/yum.conf
 
 {% endblock %}
 
-#### BEGIN REPO ENABLEMENT
+# Install what is needed for en_US.UTF-8
+{% block base_centos_distro_sync_and_languages %}
+
+{% if distro_package_manager == 'dnf' %}
+    {% set base_centos_language_packages = [
+        'langpacks-en',
+        'glibc-all-langpacks'
+    ] %}
+
+RUN {{ macros.install_packages(base_centos_language_packages | customizable("centos_language_packages"), chain=True, clean=False) }} \
+    && {{ macros.rpm_security_update(clean_package_cache) }}
+{% endif %}
+
+{% endblock %}
 
+#### BEGIN REPO ENABLEMENT
 {% set base_yum_repo_files = [
  ] %}
 
@@ -72,44 +88,71 @@ COPY yum.conf /etc/yum.conf
 
 {% if base_arch == 'x86_64' %}
     {% set base_yum_repo_files = [
-        'crmsh.repo',
-        'elasticsearch.repo',
         'grafana.repo',
         'influxdb.repo',
-        'opendaylight.repo',
         'rabbitmq_rabbitmq-server.repo',
-        'td.repo'
+        'td.repo',
     ] %}
+    # FIXME(mgoddard): Not available for CentOS 8 yet.
+    {% if distro_package_manager == 'yum' %}
+        {% set base_yum_repo_files = base_yum_repo_files + [
+            'crmsh.repo',
+            'elasticsearch.repo',
+            'opendaylight.repo',
+        ] %}
+    {% endif %}
 
     {% set base_yum_repo_keys = [
-        'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
         'https://packages.grafana.com/gpg.key',
         'https://repos.influxdata.com/influxdb.key',
-        'https://packagecloud.io/gpg.key',
-        'https://packages.treasuredata.com/GPG-KEY-td-agent'
+        'https://packages.treasuredata.com/GPG-KEY-td-agent',
     ] %}
+    # FIXME(mgoddard): Not available for CentOS 8 yet.
+    {% if distro_package_manager == 'yum' %}
+        {% set base_yum_repo_keys = base_yum_repo_keys + [
+            'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
+        ] %}
+    {% endif %}
 {% elif base_arch == 'aarch64' %}
     {% set base_yum_repo_files = [
-        'elasticsearch.repo',
         'grafana.repo',
-        'rabbitmq_rabbitmq-server.repo'
+        'rabbitmq_rabbitmq-server.repo',
     ] %}
+    # FIXME(mgoddard): Not available for CentOS 8 yet.
+    {% if distro_package_manager == 'yum' %}
+        {% set base_yum_repo_files = base_yum_repo_files + [
+            'elasticsearch.repo',
+        ] %}
+    {% endif %}
 
     {% set base_yum_repo_keys = [
-        'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
         'https://packages.grafana.com/gpg.key',
-        'https://packagecloud.io/gpg.key',
     ] %}
+    # FIXME(mgoddard): Not available for CentOS 8 yet.
+    {% if distro_package_manager == 'yum' %}
+        {% set base_yum_repo_keys = base_yum_repo_keys + [
+            'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
+        ] %}
+    {% endif %}
 {% elif base_arch == 'ppc64le' %}
     {% set base_yum_repo_files = [
-        'elasticsearch.repo',
-        'rabbitmq_rabbitmq-server.repo'
+        'rabbitmq_rabbitmq-server.repo',
     ] %}
+    # FIXME(mgoddard): Not available for CentOS 8 yet.
+    {% if distro_package_manager == 'yum' %}
+        {% set base_yum_repo_files = base_yum_repo_files + [
+            'elasticsearch.repo',
+        ] %}
+    {% endif %}
 
     {% set base_yum_repo_keys = [
-        'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
-        'https://packagecloud.io/gpg.key',
     ] %}
+    # FIXME(mgoddard): Not available for CentOS 8 yet.
+    {% if distro_package_manager == 'yum' %}
+        {% set base_yum_repo_keys = base_yum_repo_keys + [
+            'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
+        ] %}
+    {% endif %}
 {% endif %}
 
 {%- for repo_file in base_yum_repo_files | customizable('yum_repo_files') %}
@@ -160,32 +203,70 @@ RUN yum-config-manager --enable rhel-7-server-rpms \
     {% if base_distro == 'centos' %}
 
 {% block base_centos_gpg_key_import %}
+{% if distro_package_manager == 'dnf' %}
+RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+{% else %}
 RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+{% endif %}
 {% endblock %}
 
 {% set base_centos_yum_repo_keys = [
-    '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud',
-    '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-OpsTools',
-    '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Storage',
-    '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization'
 ] %}
+# FIXME(mgoddard): Not available for CentOS 8 yet.
+{% if distro_package_manager == 'yum' %}
+    {% set base_centos_yum_repo_keys = base_centos_yum_repo_keys + [
+        '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud',
+        '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-OpsTools',
+        '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Storage',
+        '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization',
+    ] %}
+{% endif %}
 
 {% set base_centos_yum_repo_packages = [
-    'centos-release-openstack-train',
-    'centos-release-ceph-nautilus',
-    'centos-release-opstools',
-    'centos-release-qemu-ev',
     'epel-release',
-    'yum-plugin-priorities'
 ] %}
+# FIXME(mgoddard): Not available for CentOS 8 yet.
+{% if distro_package_manager == 'yum' %}
+    {% set base_centos_yum_repo_packages = base_centos_yum_repo_packages + [
+        'centos-release-openstack-train',
+        'centos-release-ceph-nautilus',
+        'centos-release-opstools',
+        'centos-release-qemu-ev',
+    ] %}
+{% endif %}
+
+{% if distro_package_manager == 'dnf' %}
+    # We need 'dnf-plugins-core' for 'dnf config-manager'
+    {% set base_centos_yum_repo_packages = base_centos_yum_repo_packages + [
+        'dnf-plugins-core'
+    ] %}
+    {% set base_centos_yum_repos_to_enable = [
+        'PowerTools'
+    ] %}
+{% else %}
+    {% set base_centos_yum_repo_packages = base_centos_yum_repo_packages + [
+        'yum-plugin-priorities',
+    ] %}
+{% endif %}
 
 {% set base_centos_yum_repos_to_disable = [
 ] %}
 
 RUN {{ macros.install_packages(base_centos_yum_repo_packages | customizable("centos_yum_repo_packages"), chain=True, clean=False) }} \
-{% for repo in base_centos_yum_repos_to_disable | customizable('centos_yum_repos_to_disable') -%}
-    && yum-config-manager --disable {{ repo }} \
+{%- for repo in base_centos_yum_repos_to_enable | customizable('centos_yum_repos_to_enable') %}
+{%- if distro_package_manager == 'dnf' %}
+    && dnf config-manager --enable {{ repo }} \
+{% else %}
+    && yum-config-manager --enable {{ repo }} \
+{% endif -%}
 {% endfor -%}
+{%- for repo in base_centos_yum_repos_to_disable | customizable('centos_yum_repos_to_disable') %}
+{%- if distro_package_manager == 'dnf' %}
+    && dnf config-manager --disable {{ repo }} \
+{%- else %}
+    && yum-config-manager --disable {{ repo }} \
+{%- endif -%}
+{%- endfor %}
 {% for key in base_centos_yum_repo_keys | customizable('centos_yum_repo_keys') -%}
     && rpm --import {{ key }} \
 {% endfor -%}
@@ -249,7 +330,11 @@ RUN yum-config-manager --enable rhel-7-server-optional-rpms \
         'python'
 ] %}
 {% endif %}
-{% if distro_package_manager == 'yum' %}
+{% if distro_package_manager == 'dnf' %}
+    {% set base_centos_binary_packages = base_centos_binary_packages + [
+        'util-linux-user',
+    ] %}
+{% else %}
     {% set base_centos_binary_packages = base_centos_binary_packages + [
         'scsi-target-utils',
     ] %}
@@ -275,7 +360,11 @@ RUN yum-config-manager --enable rhel-7-server-optional-rpms \
     'tar',
     'which'
 ] %}
-{% if distro_package_manager == 'yum' %}
+{% if distro_package_manager == 'dnf' %}
+    {% set base_centos_source_packages = base_centos_source_packages + [
+        'util-linux-user',
+    ] %}
+{% else %}
     {% set base_centos_source_packages = base_centos_source_packages + [
         'scsi-target-utils',
     ] %}
diff --git a/docker/base/grafana.repo b/docker/base/grafana.repo
index f4b6f1f3f..851b9199f 100644
--- a/docker/base/grafana.repo
+++ b/docker/base/grafana.repo
@@ -1,7 +1,6 @@
 [grafana]
 name=grafana
 baseurl=https://packages.grafana.com/oss/rpm
-repo_gpgcheck=1
 enabled=1
 gpgcheck=1
 gpgkey=https://packages.grafana.com/gpg.key
diff --git a/docker/base/rabbitmq_rabbitmq-server.repo b/docker/base/rabbitmq_rabbitmq-server.repo
index 82d55cf84..b4f5ea82e 100644
--- a/docker/base/rabbitmq_rabbitmq-server.repo
+++ b/docker/base/rabbitmq_rabbitmq-server.repo
@@ -1,7 +1,10 @@
 [rabbitmq_rabbitmq-server]
 name=rabbitmq_rabbitmq-server
-baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch
-repo_gpgcheck=1
+baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/$releasever/$basearch
+# NOTE(mgoddard): rabbitmq-server 3.7 packages are not signed. Previously
+# repo_gpgcheck was set to 1, but this breaks DNF which does not seem to accept
+# keys imported by rpm --import for signing a repo (as opposed to packages).
+# TODO(mgoddard): Set gpgcheck=1 for rabbitmq-server 3.8+
 gpgcheck=0
 enabled=1
 gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
diff --git a/docker/base/td.repo b/docker/base/td.repo
index f4ccce31e..ffa5ee79f 100644
--- a/docker/base/td.repo
+++ b/docker/base/td.repo
@@ -1,5 +1,5 @@
 [treasuredata]
 name=TreasureData
-baseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch
+baseurl=http://packages.treasuredata.com/3/redhat/$releasever/$basearch
 gpgcheck=1
 gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent
diff --git a/docker/macros.j2 b/docker/macros.j2
index 0e4078ec9..f7d74ec72 100644
--- a/docker/macros.j2
+++ b/docker/macros.j2
@@ -19,7 +19,11 @@
 {# Be Extra vigilant about introducing any extra whitespace at the
    end of the macro for chaining purposes -#}
 {% macro rpm_security_update(cleanup) -%}
+{% if distro_package_manager == 'dnf' -%}
+    {{ distro_package_manager }} -y distro-sync --security --sec-severity=Important --sec-severity=Critical
+{%- else -%}
     {{ distro_package_manager }} -y update --security --sec-severity=Important --sec-severity=Critical
+{%- endif %}
     {%- if cleanup %} \
     && {{ distro_package_manager }} clean all && rm -rf /var/cache/{{ distro_package_manager }}{% endif -%}
 {%- endmacro %}
diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2
index 691e1864b..7b276bdc3 100644
--- a/docker/openstack-base/Dockerfile.j2
+++ b/docker/openstack-base/Dockerfile.j2
@@ -95,7 +95,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python3-vitrageclient',
             'python3-webob',
             'python3-zaqarclient',
-            'util-linux-user'
         ] %}
         {% else %}
         {% set openstack_base_packages = openstack_base_packages + [
@@ -241,10 +240,18 @@ ENV DEBIAN_FRONTEND noninteractive
             'openssl-devel',
             'postgresql',
             'postgresql-devel',
-            'python-devel',
             'sqlite-devel',
             'zip'
         ] %}
+        {% if distro_python_version.startswith('3') %}
+            {% set openstack_base_packages = openstack_base_packages + [
+                'python3-devel',
+            ] %}
+        {% else %}
+            {% set openstack_base_packages = openstack_base_packages + [
+                'python-devel',
+            ] %}
+        {% endif %}
 
     {% elif base_package_type == 'deb' %}
         {% set openstack_base_packages = [
diff --git a/kolla/common/config.py b/kolla/common/config.py
index 57f08d22c..bd664db9c 100755
--- a/kolla/common/config.py
+++ b/kolla/common/config.py
@@ -13,6 +13,7 @@
 import itertools
 import os
 
+from distutils.version import LooseVersion
 from oslo_config import cfg
 from oslo_config import types
 
@@ -39,6 +40,10 @@ OPENSTACK_RELEASE = 'train'
 DELOREAN = \
     "https://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo"
 DELOREAN_DEPS = "https://trunk.rdoproject.org/centos7/delorean-deps.repo"
+DELOREAN_CENTOS8 = "https://trunk.rdoproject.org/centos8-train/" \
+    "consistent/delorean.repo"
+DELOREAN_DEPS_CENTOS8 = "https://trunk.rdoproject.org/centos8-train/" \
+    "delorean-deps.repo"
 
 INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos']
 
@@ -1265,6 +1270,12 @@ def parse(conf, args, usage=None, prog=None,
     # NOTE(jeffrey4l): set the default base tag based on the
     # base option
     conf.set_default('base_tag', DEFAULT_BASE_TAGS.get(conf.base))
+    # TODO(mgoddard): Remove this 'if' when CentOS 7 is no longer supported.
+    if conf.base in ['centos', 'rhel']:
+        if LooseVersion(conf.base_tag) >= LooseVersion('8'):
+            # Use CentOS 8 Delorean repos.
+            conf.set_default('rpm_setup_config', [DELOREAN_CENTOS8,
+                                                  DELOREAN_DEPS_CENTOS8])
 
     if not conf.base_image:
         conf.base_image = conf.base
diff --git a/kolla/image/build.py b/kolla/image/build.py
index 0bdcfb5e3..b956c9995 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -144,9 +144,110 @@ UNBUILDABLE_IMAGES = {
         "ovsdpdk",
     },
 
-    # NOTE(mgoddard): Mark all images unbuildable for CentOS 8.
+    # NOTE(mgoddard): Mark all images that depend on base or openstack-base as
+    # unbuildable for CentOS 8.
     'centos8': {
-        "base",
+        "almanach-base",
+        "aodh-base",
+        "barbican-base",
+        "bifrost-base",
+        "blazar-base",
+        "ceilometer-base",
+        "ceph-base",
+        "certmonger",
+        "chrony",
+        "cinder-base",
+        "cloudkitty-base",
+        "collectd",
+        "congress-base",
+        "cron",
+        "cyborg-base",
+        "designate-base",
+        "dind",
+        "dnsmasq",
+        "dragonflow-base",
+        "ec2-api",
+        "elasticsearch",
+        "elasticsearch-curator",
+        "etcd",
+        "fluentd",
+        "freezer-base",
+        "glance-base",
+        "gnocchi-base",
+        "grafana",
+        "hacluster-base",
+        "haproxy",
+        "heat-base",
+        "helm-repository",
+        "horizon",
+        "influxdb",
+        "ironic-base",
+        "ironic-inspector",
+        "iscsid",
+        "kafka",
+        "karbor-base",
+        "keepalived",
+        "keystone-base",
+        "kibana",
+        "kolla-toolbox",
+        "kube-base",
+        "kubernetes-entrypoint",
+        "kubetoolbox",
+        "kuryr-base",
+        "logstash",
+        "magnum-base",
+        "manila-base",
+        "mariadb",
+        "masakari-base",
+        "memcached",
+        "mistral-base",
+        "monasca-base",
+        "monasca-grafana",
+        "monasca-thresh",
+        "mongodb",
+        "multipathd",
+        "murano-base",
+        "networking-baremetal",
+        "neutron-base",
+        "nova-base",
+        "nova-libvirt",
+        "novajoin-base",
+        "octavia-base",
+        "opendaylight",
+        "openvswitch-base",
+        "ovn-base",
+        "ovsdpdk",
+        "panko-base",
+        "placement-base",
+        "prometheus-base",
+        "ptp",
+        "qdrouterd",
+        "qinling-base",
+        "rabbitmq",
+        "radvd",
+        "rally",
+        "redis-base",
+        "rsyslog-base",
+        "sahara-base",
+        "searchlight-base",
+        "senlin-base",
+        "sensu-base",
+        "skydive-base",
+        "solum-base",
+        "storm",
+        "swift-base",
+        "tacker-base",
+        "telegraf",
+        "tempest",
+        "tgtd",
+        "tripleoclient",
+        "trove-base",
+        "vitrage-base",
+        "vmtp",
+        "watcher-base",
+        "zaqar-base",
+        "zookeeper",
+        "zun-base",
     },
 
     'debian': {
diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2
index d907a8dcd..3b88ba8e4 100644
--- a/tests/templates/template_overrides.j2
+++ b/tests/templates/template_overrides.j2
@@ -29,13 +29,20 @@ RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-
 RUN sed -i \
         -e "/^mirrorlist/ d" \
         -e "s|^#baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \
-        /etc/yum.repos.d/CentOS-*.repo \
-    && sed -i \
+        /etc/yum.repos.d/CentOS-*.repo
+{% raw %}
+{% if distro_package_manager == 'yum' %}
+{% endraw %}
+# FIXME(mgoddard): No crmsh or openstacklight CentOS 8 repos yet.
+RUN sed -i \
         -e "s|http://download.opensuse.org|http://{{ nodepool_mirror_host }}/opensuse|" \
         /etc/yum.repos.d/crmsh.repo \
     && sed -i \
         -e "s|https://cbs.centos.org/repos|{{ nodepool_cbs_centos_proxy }}|g" \
         /etc/yum.repos.d/opendaylight.repo
+{% raw %}
+{% endif %}
+{% endraw %}
 
 {% raw %}
 {% endblock %}
@@ -55,6 +62,7 @@ RUN sed -i \
     && sed -i \
         -e "/^metalink/ d" \
         -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \
+        -e "s|^#baseurl=https://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \
         /etc/yum.repos.d/epel*.repo \
 {% raw %}
 {% endblock %}