CI: make it work again

Python changes:
- limit pip to 20.x as this is last version working on both Python 2 and 3.
- cap 'setuptools' to be older than 58 to have use_2to3 functionality
  (fixes 'anyjson' installation)
- upgrade pip in 'elasticsearch-curator' image to get it built on CentOS 7

Unbuildable images:
- 'bifrost-base' under CentOS 8
  (Error: Cannot find a valid baseurl for repo: epel)
- 'monasca-grafana' under CentOS 7/8
- 'sensu-client' under CentOS 7
  (aws-eventstream requires Ruby version >= 2.3)

- dropped *-upgrade CI jobs as Stein is EOL

- pin 'qpid-proton-c' package for CentOS 7 to not install EPEL one
  (cinder-volume, openstack-base and tripleoclient)

- bump 'libvirt-python' in 'masakari-monitors' to 7.10.0
  (newer branches install package from distribution but CentOS 7 lacks
  it)

- pin 'unf_ext' gem in Sensu images as CentOS 7 lacks Ruby 2.2+
  (fixed sensu-server)

- install 'ca-certificates' for newer Let's Encrypt certificates support
  (so we can fetch upper-constraints)

- make Zuul fetch openstack/requirements project files
   (make 'openstack-base' pass) (like 2nd fix)

Co-Authored-by: Pierre Riteau <pierre@stackhpc.com>
Closes-Bug: #1943617
Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/826464

Change-Id: I8ee32e53cda581493b2a4bb73ca72ab05014b9d4

7 files changed, 17 insertions, 6 deletions

diff --git a/docker/cinder/cinder-volume/Dockerfile.j2 b/docker/cinder/cinder-volume/Dockerfile.j2
index a5f3af756..a06432475 100644
--- a/docker/cinder/cinder-volume/Dockerfile.j2
+++ b/docker/cinder/cinder-volume/Dockerfile.j2
@@ -18,9 +18,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python3-cinderlib',
     ] %}
     {% else %}
+    {# NOTE(hrw): qpid-proton-c 0.26 from centos-openstack-train repo instead of epel #}
     {% set cinder_volume_packages = cinder_volume_packages + [
             'python-rtslib',
             'python2-cinderlib',
+            'qpid-proton-c-0.26.*',
     ] %}
     {% endif %}
 {{ macros.install_packages(cinder_volume_packages | customizable("packages")) }}
diff --git a/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2 b/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2
index 0c2588ecc..2f7950761 100644
--- a/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2
+++ b/docker/elasticsearch/elasticsearch-curator/Dockerfile.j2
@@ -27,7 +27,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     'elasticsearch-curator'
 ] %}
 
-RUN {{ macros.install_pip(elasticsearch_curator_pip_packages | customizable("pip_packages"), constraints=false) }} \
+# NOTE(hrw): CentOS 7 has pip 8 which fails to handle boto3/botocore requirements of elasticsearch-curator
+RUN {{ macros.install_pip(['pip==20.*'], constraints=false) }} \
+    && {{ macros.install_pip(elasticsearch_curator_pip_packages | customizable("pip_packages"), constraints=false) }} \
     && mkdir -p /etc/elasticsearch-curator \
     && chown -R elasticsearch: /etc/elasticsearch-curator
 
diff --git a/docker/masakari/masakari-monitors/Dockerfile.j2 b/docker/masakari/masakari-monitors/Dockerfile.j2
index b50951141..7bf5ac97a 100644
--- a/docker/masakari/masakari-monitors/Dockerfile.j2
+++ b/docker/masakari/masakari-monitors/Dockerfile.j2
@@ -41,7 +41,7 @@ ADD masakari-monitors-archive /masakari-monitors-source
 ] %}
 
 RUN ln -s masakari-monitors-source/* masakari-monitors \
-    {% if distro_package_manager == 'dnf' %}&& sed -i -e 's/libvirt-python===.*/libvirt-python===6.10.0/' /requirements/upper-constraints.txt {% endif %}\
+    {% if distro_package_manager == 'dnf' %}&& sed -i -e 's/libvirt-python===.*/libvirt-python===7.10.0/' /requirements/upper-constraints.txt {% endif %}\
     && {{ macros.install_pip(masakari_monitors_pip_packages | customizable("pip_packages")) }} \
     && mkdir -p /etc/masakari-monitors \
     && chown -R masakari: /etc/masakari-monitors
diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2
index 9ff44b86f..408a2af63 100644
--- a/docker/openstack-base/Dockerfile.j2
+++ b/docker/openstack-base/Dockerfile.j2
@@ -96,6 +96,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python3-zaqarclient',
         ] %}
         {% else %}
+        {# NOTE(hrw): qpid-proton-c 0.26 from centos-openstack-train repo instead of epel #}
         {% set openstack_base_packages = openstack_base_packages + [
             'python2-cryptography',
             'python2-eventlet',
@@ -173,6 +174,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python2-webob',
             'python-zaqarclient',
             'PyYAML',
+            'qpid-proton-c-0.26.*',
             'sysvinit-tools'
         ] %}
         {% endif %}
@@ -410,7 +412,9 @@ RUN ln -s openstack-base-source/* /requirements \
 
 ENV PATH /var/lib/kolla/venv/bin:$PATH
 
-RUN {{ macros.install_pip(openstack_base_pip_packages | customizable("pip_packages")) }}
+{# NOTE(hrw): setuptools is capped in openstack/requirements as this version still has use_2to3 #}
+RUN {{ macros.install_pip(["'setuptools < 58'"]) }} \
+ && {{ macros.install_pip(openstack_base_pip_packages | customizable("pip_packages")) }}
 
 {% endif %}
 
diff --git a/docker/sensu/sensu-client/Dockerfile.j2 b/docker/sensu/sensu-client/Dockerfile.j2
index 7e9580432..f0317d7b3 100644
--- a/docker/sensu/sensu-client/Dockerfile.j2
+++ b/docker/sensu/sensu-client/Dockerfile.j2
@@ -89,7 +89,8 @@ RUN echo '{{ image_name }} not yet available for {{ base_distro }}' \
 # NOTE(hrw): whois 5.0.0 requires Ruby 2.4+ while CentOS has 2.0
 # NOTE(yoctozepto): pinning minitest for the same reason
 # NOTE(yoctozepto): pinning simpleidn because it started requiring Ruby 2.2
-RUN {%if base_package_type == 'rpm' %} gem install whois:"<5" minitest:"~>5.11.3" simpleidn:"~>0.1.1" && {% endif %} sensu-install --plugins {{ sensu_plugins | customizable('plugins') | join (',') }}
+# NOTE(hrw): gem order is crucial - simpleidn depends on unf_ext
+RUN {%if base_package_type == 'rpm' %} gem install whois:"<5" minitest:"~>5.11.3" unf_ext:"<0.0.7.7" simpleidn:"~>0.1.1" && {% endif %} sensu-install --plugins {{ sensu_plugins | customizable('plugins') | join (',') }}
 {% endblock %}
 
 {% block sensu_client_footer %}{% endblock %}
diff --git a/docker/sensu/sensu-server/Dockerfile.j2 b/docker/sensu/sensu-server/Dockerfile.j2
index fa447fe7c..543262de8 100644
--- a/docker/sensu/sensu-server/Dockerfile.j2
+++ b/docker/sensu/sensu-server/Dockerfile.j2
@@ -31,7 +31,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     'slack:"~>3"'
 ] %}
 
-RUN sensu-install --plugins {{ sensu_plugins | customizable('plugins') | join (',') }}
+RUN {%if base_package_type == 'rpm' %} gem install unf_ext:"<0.0.7.7" && {% endif %} sensu-install --plugins {{ sensu_plugins | customizable('plugins') | join (',') }}
 {% endblock %}
 
 {% block sensu_server_footer %}{% endblock %}
diff --git a/docker/tripleoclient/Dockerfile.j2 b/docker/tripleoclient/Dockerfile.j2
index b47eff638..770d5f3cc 100644
--- a/docker/tripleoclient/Dockerfile.j2
+++ b/docker/tripleoclient/Dockerfile.j2
@@ -22,10 +22,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
                 'python3-tripleoclient'
         ] %}
         {% else %}
+        {# NOTE(hrw): qpid-proton-c 0.26 from centos-openstack-train repo instead of epel #}
         {% set tripleoclient_packages = tripleoclient_packages + [
                 'python-ipaddr',
                 'python-openstackclient',
-                'python-tripleoclient'
+                'python-tripleoclient',
+                'qpid-proton-c-0.26.*'
         ] %}
         {% endif %}