CentOS 8: Update packages in images

* Some further changes for python2 vs python3 packages
* Allow rabbitmq 3.7.*, since a newer erlang is available
* Switch from qemu-img-ev to qemu-img on CentOS 8
* bridge-utils no longer available on CentOS 8
* libvirt-daemon-driver-lxc no longer available on CentOS 8
* Mark some more images buildable for CentOS 8

Change-Id: Iaf5b68ff6d944ae730ca0b1d5832172c106a6c08
Partially-Implements: blueprint centos-rhel-8
Partially-Implements: blueprint centos-rhel-python-3

18 files changed, 165 insertions, 128 deletions

diff --git a/docker/ceilometer/ceilometer-compute/Dockerfile.j2 b/docker/ceilometer/ceilometer-compute/Dockerfile.j2
index e587ad861..bd0aa4c6b 100644
--- a/docker/ceilometer/ceilometer-compute/Dockerfile.j2
+++ b/docker/ceilometer/ceilometer-compute/Dockerfile.j2
@@ -20,9 +20,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {% elif install_type == 'source' %}
     {% if base_package_type == 'rpm' %}
-        {% set ceilometer_compute_packages = [
-            'libvirt-python'
-         ] %}
+        {% if distro_python_version.startswith('3') %}
+            {% set ceilometer_compute_packages = [
+                'python3-libvirt'
+             ] %}
+        {% else %}
+            {% set ceilometer_compute_packages = [
+                'libvirt-python'
+             ] %}
+        {% endif %}
     {% elif base_package_type == 'deb' %}
         {% set ceilometer_compute_packages = [
             'python3-libvirt'
diff --git a/docker/cinder/cinder-base/Dockerfile.j2 b/docker/cinder/cinder-base/Dockerfile.j2
index a20902f0f..8ed7e0d20 100644
--- a/docker/cinder/cinder-base/Dockerfile.j2
+++ b/docker/cinder/cinder-base/Dockerfile.j2
@@ -50,13 +50,13 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
                 'lvm2',
                 'cryptsetup',
         ] %}
-        {% if base_distro in ['centos'] %}
+        {% if distro_package_manager == 'dnf' or base_distro not in ['centos'] %}
             {% set cinder_base_packages = cinder_base_packages + [
-                'qemu-img-ev'
+                'qemu-img'
             ] %}
         {% else %}
             {% set cinder_base_packages = cinder_base_packages + [
-                 'qemu-img'
+                 'qemu-img-ev'
             ] %}
         {% endif %}
     {% elif base_package_type == 'deb' %}
diff --git a/docker/cinder/cinder-volume/Dockerfile.j2 b/docker/cinder/cinder-volume/Dockerfile.j2
index a65718bbc..77748fd1a 100644
--- a/docker/cinder/cinder-volume/Dockerfile.j2
+++ b/docker/cinder/cinder-volume/Dockerfile.j2
@@ -12,7 +12,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'sysfsutils',
             'targetcli'
     ] %}
-
     {% if distro_python_version.startswith('3') %}
     {% set cinder_volume_packages = cinder_volume_packages + [
             'python3-rtslib',
diff --git a/docker/glance/glance-api/Dockerfile.j2 b/docker/glance/glance-api/Dockerfile.j2
index 9ac24070a..1b98f1efe 100644
--- a/docker/glance/glance-api/Dockerfile.j2
+++ b/docker/glance/glance-api/Dockerfile.j2
@@ -5,10 +5,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {% import "macros.j2" as macros with context %}
 
-{% if base_distro in ['centos'] %}
-    {% set glance_api_packages = ['qemu-img-ev'] %}
-{% elif base_distro == 'rhel' %}
+{% if base_package_type == 'rpm' %}
+    {% if distro_package_manager == 'dnf' or base_distro not in ['centos'] %}
     {% set glance_api_packages = ['qemu-img'] %}
+    {% else %}
+    {% set glance_api_packages = ['qemu-img-ev'] %}
+    {% endif %}
 {% elif base_package_type == 'deb' %}
     {% set glance_api_packages = ['qemu-utils'] %}
 {% endif %}
diff --git a/docker/ironic/ironic-conductor/Dockerfile.j2 b/docker/ironic/ironic-conductor/Dockerfile.j2
index 21cc6bdc3..dece02e25 100644
--- a/docker/ironic/ironic-conductor/Dockerfile.j2
+++ b/docker/ironic/ironic-conductor/Dockerfile.j2
@@ -49,13 +49,13 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'systemd-python'
         ] %}
         {% endif %}
-        {% if base_distro in ['centos'] %}
+        {% if distro_package_manager == 'dnf' or base_distro not in ['centos'] %}
             {% set ironic_conductor_packages = ironic_conductor_packages + [
-                'qemu-img-ev'
+                'qemu-img'
             ] %}
         {% else %}
             {% set ironic_conductor_packages = ironic_conductor_packages + [
-                'qemu-img'
+                'qemu-img-ev'
             ] %}
         {% endif %}
     {% elif base_package_type == 'deb' %}
@@ -92,7 +92,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'ipmitool',
             'parted',
             'psmisc',
-            'qemu-img-ev',
             'shellinabox',
             'util-linux',
             'xfsprogs',
@@ -106,6 +105,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'systemd',
         ] %}
         {% endif %}
+        {% if distro_package_manager == 'dnf' or base_distro not in ['centos'] %}
+            {% set ironic_conductor_packages = ironic_conductor_packages + [
+                'qemu-img'
+            ] %}
+        {% else %}
+            {% set ironic_conductor_packages = ironic_conductor_packages + [
+                'qemu-img-ev'
+            ] %}
+        {% endif %}
     {% elif base_package_type == 'deb' %}
         {% set ironic_conductor_packages = [
             'bsdmainutils',
diff --git a/docker/keystone/keystone-base/Dockerfile.j2 b/docker/keystone/keystone-base/Dockerfile.j2
index 5a4e8eddf..85d0734eb 100644
--- a/docker/keystone/keystone-base/Dockerfile.j2
+++ b/docker/keystone/keystone-base/Dockerfile.j2
@@ -31,6 +31,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         ] %}
         {% endif %}
 
+{% if distro_package_manager == 'dnf' %}
+RUN dnf module enable mod_auth_openidc -y
+{% endif %}
+
 {{ macros.install_packages(keystone_base_packages | customizable("packages")) }}
 RUN mkdir -p /var/www/cgi-bin/keystone \
     && cp -a /usr/bin/keystone-wsgi-public /var/www/cgi-bin/keystone/main \
@@ -76,6 +80,10 @@ RUN mkdir -p /var/www/cgi-bin/keystone \
         ] %}
         {% endif %}
 
+{% if distro_package_manager == 'dnf' %}
+RUN dnf module enable mod_auth_openidc -y
+{% endif %}
+
 {{ macros.install_packages(keystone_base_packages | customizable("packages")) }}
 RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
     && sed -i -r 's,^(Listen 443),#\1,' /etc/httpd/conf.d/ssl.conf
diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2
index d4ad59f03..6486e882d 100644
--- a/docker/kolla-toolbox/Dockerfile.j2
+++ b/docker/kolla-toolbox/Dockerfile.j2
@@ -30,9 +30,20 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'mariadb-devel',
         'openssh-clients',
         'openssl-devel',
-        'python-devel',
-        'rabbitmq-server-3.7.10',
      ] %}
+    {% if distro_package_manager == 'dnf' %}
+        {% set kolla_toolbox_packages = kolla_toolbox_packages + [
+            'python3-devel',
+            'rabbitmq-server-3.7.*'
+        ] %}
+    {% else %}
+        # NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
+        # https://bugs.launchpad.net/kolla/+bug/1814233
+        {% set kolla_toolbox_packages = kolla_toolbox_packages + [
+            'python-devel',
+            'rabbitmq-server-3.7.10'
+        ] %}
+    {% endif %}
 
 {% elif base_package_type == 'deb' %}
     {% set kolla_toolbox_packages = [
diff --git a/docker/kuryr/kuryr-libnetwork/Dockerfile.j2 b/docker/kuryr/kuryr-libnetwork/Dockerfile.j2
index ccfbf9ffb..c5a63be02 100644
--- a/docker/kuryr/kuryr-libnetwork/Dockerfile.j2
+++ b/docker/kuryr/kuryr-libnetwork/Dockerfile.j2
@@ -14,9 +14,13 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
     {% if base_package_type == 'rpm' %}
 
         {% set kuryr_libnetwork_packages = [
-            'bridge-utils',
             'openvswitch'
         ] %}
+        {% if distro_package_manager == 'yum' %}
+        {% set kuryr_libnetwork_packages = kuryr_libnetwork_packages + [
+            'bridge-utils',
+        ] %}
+        {% endif %}
 
     {% elif base_package_type == 'deb' %}
 
diff --git a/docker/neutron/neutron-base/Dockerfile.j2 b/docker/neutron/neutron-base/Dockerfile.j2
index d29597119..aa4c2a854 100644
--- a/docker/neutron/neutron-base/Dockerfile.j2
+++ b/docker/neutron/neutron-base/Dockerfile.j2
@@ -83,14 +83,23 @@ RUN mkdir -p /usr/share/neutron \
             'dnsmasq-utils',
             'haproxy',
             'ipset',
+            'iptables',
             'iputils',
             'keepalived',
             'net-tools',
             'openvswitch',
-            'python2-openvswitch',
             'radvd',
             'uuid'
         ] %}
+        {% if distro_python_version.startswith('3') %}
+            {% set neutron_base_packages = neutron_base_packages + [
+                'python3-openvswitch',
+            ] %}
+        {% else %}
+            {% set neutron_base_packages = neutron_base_packages + [
+                'python2-openvswitch',
+            ] %}
+        {% endif %}
 
     {% elif base_package_type == 'deb' %}
 
diff --git a/docker/neutron/neutron-linuxbridge-agent/Dockerfile.j2 b/docker/neutron/neutron-linuxbridge-agent/Dockerfile.j2
index a486fd5f9..514e88f3e 100644
--- a/docker/neutron/neutron-linuxbridge-agent/Dockerfile.j2
+++ b/docker/neutron/neutron-linuxbridge-agent/Dockerfile.j2
@@ -25,9 +25,13 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% elif install_type == 'source' %}
 
     {% set neutron_linuxbridge_agent_packages = [
-        'bridge-utils',
         'ebtables'
     ] %}
+    {% if distro_package_manager != 'dnf' %}
+        {% set neutron_linuxbridge_agent_packages = neutron_linuxbridge_agent_packages + [
+            'bridge-utils',
+        ] %}
+    {% endif %}
 
 {% endif %}
 
diff --git a/docker/nova/nova-compute/Dockerfile.j2 b/docker/nova/nova-compute/Dockerfile.j2
index a8e103694..ff41c0fbe 100644
--- a/docker/nova/nova-compute/Dockerfile.j2
+++ b/docker/nova/nova-compute/Dockerfile.j2
@@ -83,18 +83,36 @@ RUN rm -f /etc/nova/nova-compute.conf
             'e2fsprogs',
             'genisoimage',
             'iscsi-initiator-utils',
-            'libvirt-python',
             'libosinfo',
             'nfs-utils',
             'openvswitch',
             'parted',
-            'python-libguestfs',
-            'python-rtslib',
-            'qemu-img-ev',
             'sysfsutils',
             'targetcli',
             'xfsprogs'
         ] %}
+        {% if distro_python_version.startswith('3') %}
+        {% set nova_compute_packages = nova_compute_packages + [
+            'python3-libguestfs',
+            'python3-libvirt',
+            'python3-rtslib'
+        ] %}
+        {% else %}
+        {% set nova_compute_packages = nova_compute_packages + [
+            'libvirt-python',
+            'python-libguestfs',
+            'python-rtslib'
+        ] %}
+        {% endif %}
+        {% if distro_package_manager == 'dnf' %}
+        {% set nova_compute_packages = nova_compute_packages + [
+            'qemu-img',
+        ] %}
+        {% else %}
+        {% set nova_compute_packages = nova_compute_packages + [
+            'qemu-img-ev',
+        ] %}
+        {% endif %}
 
     {% elif base_package_type == 'deb' %}
 
diff --git a/docker/nova/nova-libvirt/Dockerfile.j2 b/docker/nova/nova-libvirt/Dockerfile.j2
index 12ca843ef..293020d60 100644
--- a/docker/nova/nova-libvirt/Dockerfile.j2
+++ b/docker/nova/nova-libvirt/Dockerfile.j2
@@ -18,18 +18,22 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'libvirt-client',
         'libvirt-daemon',
         'libvirt-daemon-config-nwfilter',
-        'libvirt-daemon-driver-lxc',
         'libvirt-daemon-driver-nwfilter',
         'openvswitch',
         'trousers'
     ] %}
-    {% if base_distro in ['centos'] %}
+    {% if distro_package_manager == 'dnf' or base_distro not in ['centos'] %}
         {% set nova_libvirt_packages = nova_libvirt_packages + [
-            'qemu-kvm-ev'
+            'qemu-kvm'
         ] %}
     {% else %}
         {% set nova_libvirt_packages = nova_libvirt_packages + [
-            'qemu-kvm'
+            'qemu-kvm-ev'
+        ] %}
+    {% endif %}
+    {% if distro_package_manager != 'dnf' %}
+        {% set nova_libvirt_packages = nova_libvirt_packages + [
+            'libvirt-daemon-driver-lxc',
         ] %}
     {% endif %}
 
diff --git a/docker/nova/nova-spicehtml5proxy/Dockerfile.j2 b/docker/nova/nova-spicehtml5proxy/Dockerfile.j2
index 57b439766..b5da34c12 100644
--- a/docker/nova/nova-spicehtml5proxy/Dockerfile.j2
+++ b/docker/nova/nova-spicehtml5proxy/Dockerfile.j2
@@ -36,10 +36,19 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% if base_package_type == 'rpm' %}
 
         {% set nova_spicehtml5proxy_packages = [
-            'numpy',
             'spice-html5'
         ] %}
 
+        {% if distro_python_version.startswith('3') %}
+        {% set nova_spicehtml5proxy_packages = nova_spicehtml5proxy_packages + [
+            'python3-numpy',
+        ] %}
+        {% else %}
+        {% set nova_spicehtml5proxy_packages = nova_spicehtml5proxy_packages + [
+            'numpy',
+        ] %}
+        {% endif %}
+
     {% elif base_package_type == 'deb' %}
 
         {% set nova_spicehtml5proxy_packages = [
diff --git a/docker/rabbitmq/Dockerfile.j2 b/docker/rabbitmq/Dockerfile.j2
index ff17be7d0..c954675e8 100644
--- a/docker/rabbitmq/Dockerfile.j2
+++ b/docker/rabbitmq/Dockerfile.j2
@@ -11,13 +11,21 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {% block rabbitmq_install %}
 {% if base_package_type == 'rpm' %}
-# NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-# https://bugs.launchpad.net/kolla/+bug/1814233
     {% set rabbitmq_packages = [
         'erlang-hipe',
         'hostname',
-        'rabbitmq-server-3.7.10'
     ] %}
+    {% if distro_package_manager == 'dnf' %}
+        {% set rabbitmq_packages = rabbitmq_packages + [
+            'rabbitmq-server-3.7.*'
+        ] %}
+    {% else %}
+        # NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
+        # https://bugs.launchpad.net/kolla/+bug/1814233
+        {% set rabbitmq_packages = rabbitmq_packages + [
+            'rabbitmq-server-3.7.10'
+        ] %}
+    {% endif %}
 
 {% elif base_package_type == 'deb' %}
     {% set rabbitmq_packages = [
diff --git a/kolla/image/build.py b/kolla/image/build.py
index 603f6e0df..a70919842 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -142,103 +142,37 @@ UNBUILDABLE_IMAGES = {
         "ovsdpdk",
     },
 
-    # NOTE(mgoddard): Mark all images that depend on base or openstack-base as
-    # unbuildable for CentOS 8.
+    # NOTE(mgoddard): Mark images with missing dependencies as unbuildable for
+    # CentOS 8.
     'centos8': {
-        "aodh-base",
-        "barbican-base",
-        "bifrost-base",
-        "blazar-base",
-        "ceilometer-base",
-        "ceph-base",
-        "certmonger",
-        "chrony",
-        "cinder-base",
-        "cloudkitty-base",
-        "collectd",
-        "congress-base",
-        "cron",
-        "cyborg-base",
-        "designate-base",
-        "dnsmasq",
-        "ec2-api",
-        "elasticsearch",
-        "elasticsearch-curator",
-        "etcd",
-        "fluentd",
-        "freezer-base",
-        "glance-base",
-        "gnocchi-base",
-        "grafana",
-        "hacluster-base",
-        "haproxy",
-        "heat-base",
-        "horizon",
-        "influxdb",
-        "ironic-base",
-        "ironic-inspector",
-        "iscsid",
-        "kafka",
-        "karbor-base",
-        "keepalived",
-        "keystone-base",
-        "kibana",
-        "kolla-toolbox",
-        "kuryr-base",
-        "logstash",
-        "magnum-base",
-        "manila-base",
-        "mariadb",
-        "masakari-base",
-        "memcached",
-        "mistral-base",
-        "monasca-base",
-        "monasca-grafana",
-        "monasca-thresh",
-        "mongodb",
-        "multipathd",
-        "murano-base",
-        "networking-baremetal",
-        "neutron-base",
-        "nova-base",
-        "nova-libvirt",
-        "novajoin-base",
-        "octavia-base",
-        "opendaylight",
-        "openvswitch-base",
-        "ovn-base",
-        "ovsdpdk",
-        "panko-base",
-        "placement-base",
-        "prometheus-base",
-        "ptp",
-        "qdrouterd",
-        "qinling-base",
-        "rabbitmq",
-        "radvd",
-        "rally",
-        "redis-base",
-        "rsyslog-base",
-        "sahara-base",
-        "searchlight-base",
-        "senlin-base",
-        "sensu-base",
-        "skydive-base",
-        "solum-base",
-        "storm",
-        "swift-base",
-        "tacker-base",
-        "telegraf",
-        "tempest",
-        "tgtd",
-        "tripleoclient",
-        "trove-base",
-        "vitrage-base",
-        "vmtp",
-        "watcher-base",
-        "zaqar-base",
-        "zookeeper",
-        "zun-base",
+        "ceph-base",             # Missing Ceph repo
+        "cinder-base",           # Missing Ceph repo
+        "collectd",              # Missing collectd-ping and
+                                 # collectd-sensubility packages
+        "elasticsearch",         # Missing elasticsearch repo
+        "glance-base",           # Missing Ceph repo
+        "gnocchi-base",          # Missing Ceph repo
+        "hacluster-base",        # Missing hacluster repo
+        "ironic-conductor",      # Missing shellinabox package
+        "kibana",                # Missing elasticsearch repo
+        "manila-share",          # Missing Ceph repo
+        "mongodb",               # Missing mongodb and mongodb-server packages
+        "monasca-grafana",       # Using python2
+        "nova-compute",          # Missing Ceph repo
+        "nova-libvirt",          # Missing Ceph repo
+        "nova-spicehtml5proxy",  # Missing spicehtml5 package
+        "opendaylight",          # Missing opendaylight repo
+        "ovsdpdk",               # Not supported on CentOS
+        "sensu-base",            # Missing sensu package
+        "tgtd",                  # Not supported on CentOS 8
+    },
+
+    'centos8+source': {
+        "bifrost-base",          # Bifrost does not support CentOS 8
+        "cyborg-agent",          # opae-sdk does not support CentOS 8
+        "freezer-base",          # Missing package trickle
+        "masakari-monitors",     # Missing hacluster repo
+        "zun-compute",           # Missing Ceph repo
     },
 
     'debian': {
diff --git a/kolla/template/methods.py b/kolla/template/methods.py
index c4c34125d..b0a91158e 100644
--- a/kolla/template/methods.py
+++ b/kolla/template/methods.py
@@ -91,6 +91,10 @@ def enable_repos(context, reponames):
     base_arch = context.get('base_arch')
     distro_package_manager = context.get('distro_package_manager')
 
+    # TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
+    if base_distro == 'centos' and distro_package_manager == 'dnf':
+        base_distro = 'centos8'
+
     commands = ''
 
     try:
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index 90b1bbd2b..0ea8a4bdd 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -1,4 +1,5 @@
 ---
+# TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
 centos:
   ceph: "centos-ceph-nautilus"
   elasticsearch: "elasticsearch-kibana-logstash-5.x"
@@ -11,6 +12,7 @@ centos:
   rabbitmq: "rabbitmq_rabbitmq-server"
   td-agent: "treasuredata"
 
+# TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
 centos-aarch64:
   ceph: "centos-ceph-nautilus"
   elasticsearch: "elasticsearch-kibana-logstash-5.x"
@@ -19,12 +21,19 @@ centos-aarch64:
   opendaylight: "opendaylight"
   rabbitmq: "rabbitmq_rabbitmq-server"
 
+# TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
 centos-ppc64le:
   ceph: "centos-ceph-nautilus"
   elasticsearch: "elasticsearch-kibana-logstash-5.x"
   nfs-ganesha: "centos-nfs-ganesha28"
   rabbitmq: "rabbitmq_rabbitmq-server"
 
+centos8:
+  grafana: "grafana"
+  influxdb: "influxdb"
+  rabbitmq: "rabbitmq_rabbitmq-server"
+  td-agent: "treasuredata"
+
 # NOTE(hrw): empty dict to avoid TypeError
 rhel: {}
 
diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2
index 87d34244f..e79d1026f 100644
--- a/tests/templates/template_overrides.j2
+++ b/tests/templates/template_overrides.j2
@@ -33,7 +33,7 @@ RUN sed -i \
 {% raw %}
 {% if distro_package_manager == 'yum' %}
 {% endraw %}
-# FIXME(mgoddard): No crmsh or openstacklight CentOS 8 repos yet.
+# FIXME(mgoddard): No crmsh or opendaylight CentOS 8 repos yet.
 RUN sed -i \
         -e "s|http://download.opensuse.org|http://{{ nodepool_mirror_host }}/opensuse|" \
         /etc/yum.repos.d/crmsh.repo \