move to Debian 'bullseye'

Bullseye is now in hard freeze cycle and goes for release.

https://release.debian.org/bullseye/freeze_policy.html

Co-Authored-By: Michal Nasiadka <mnasiadka@gmail.com>

Change-Id: I543965a2741cebfa759576a4c75669a7bacd4208

14 files changed, 112 insertions, 102 deletions

diff --git a/.zuul.d/debian.yaml b/.zuul.d/debian.yaml
index 0ae1ef5de..b1a32dac0 100644
--- a/.zuul.d/debian.yaml
+++ b/.zuul.d/debian.yaml
@@ -4,11 +4,15 @@
       jobs:
         - kolla-build-debian-source
         - kolla-build-debian-binary
-        - kolla-ansible-debian-source
+        - kolla-ansible-debian-source:
+            vars:
+              kolla_python_version: 3.9
     check-arm64:
       jobs:
         - kolla-build-debian-source-aarch64
-        - kolla-ansible-debian-source-aarch64
+        - kolla-ansible-debian-source-aarch64:
+            vars:
+              kolla_python_version: 3.9
     periodic:
       jobs:
         - kolla-publish-debian-source-quay
diff --git a/doc/source/matrix_aarch64.csv b/doc/source/matrix_aarch64.csv
index e7d4bfcef..9b4ed7a4d 100644
--- a/doc/source/matrix_aarch64.csv
+++ b/doc/source/matrix_aarch64.csv
@@ -32,14 +32,14 @@ iscsid,C,C,C,C,N,C
 kafka,C,C,C,C,N,C
 keepalived,C,C,C,C,N,C
 keystone,C,C,C,C,N,C
-kibana,N,N,N,N,C,C
+kibana,N,N,N,N,N,C
 kolla-toolbox,C,C,C,C,N,C
 kuryr,N,C,N,C,N,C
 logstash,C,C,C,C,N,C
 magnum,C,C,C,C,N,C
 manila,C,C,C,C,N,C
 mariadb,C,C,C,C,N,C
-masakari,N,C,C,C,C,C
+masakari,N,C,C,C,N,C
 memcached,C,C,C,C,N,C
 mistral,C,C,C,C,N,C
 monasca,N,N,N,N,N,N
@@ -51,8 +51,8 @@ nova,C,C,C,C,N,C
 nova-spicehtml5proxy,N,N,C,C,N,C
 octavia,C,C,N,C,N,C
 openvswitch,C,C,C,C,N,C
-ovn,C,C,C,C,N,C
-ovsdpdk,N,N,C,C,N,N
+ovn,C,C,C,C,N,N
+ovsdpdk,N,N,C,C,N,C
 panko,C,C,C,C,N,C
 placement,C,C,N,C,N,C
 prometheus,C,C,C,C,N,C
diff --git a/doc/source/matrix_x86.csv b/doc/source/matrix_x86.csv
index 89b20adfb..5a423ca43 100644
--- a/doc/source/matrix_x86.csv
+++ b/doc/source/matrix_x86.csv
@@ -47,13 +47,13 @@ monasca,N,C,N,C,N,N
 multipathd,C,C,C,C,C,C
 murano,C,C,C,C,C,C
 neutron,T,T,T,T,C,T
-neutron-mlnx-agent,C,C,N,C,N,C
+neutron-mlnx-agent,C,C,N,C,C,C
 nova,T,T,T,T,C,T
 nova-spicehtml5proxy,N,N,T,T,C,T
 octavia,C,C,N,C,C,C
 openvswitch,T,T,T,T,C,T
-ovn,C,C,C,C,C,C
-ovsdpdk,N,N,C,C,N,N
+ovn,C,C,C,C,N,N
+ovsdpdk,N,N,C,C,C,C
 panko,C,C,C,C,C,C
 placement,T,T,T,T,C,T
 prometheus,C,C,C,C,C,C
diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index fc55949ee..6c5d9df83 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -328,6 +328,18 @@ COPY sources.list /etc/apt/sources.list.d/kolla-custom.list
 
 {% block base_debian_after_sources_list %}{% endblock %}
 
+{# install Debian Openstack repos - they are not mirrored on CI #}
+{% if base_distro == 'debian' %}
+
+RUN apt update \
+ && apt install -y --no-install-recommends extrepo \
+ && extrepo enable openstack_wallaby \
+ && apt purge -y extrepo \
+ && apt --purge autoremove -y \
+ && apt clean
+
+{% endif %}
+
 {% block base_ubuntu_package_apt_preferences %}
 COPY apt_preferences.{{ base_distro }} /etc/apt/preferences
 COPY apt_preferences /etc/apt/preferences.d/kolla-custom
@@ -383,10 +395,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
     {% set remote_apt_keys = [
       'https://download.docker.com/linux/debian/gpg',
       'https://packages.grafana.com/gpg.key',
-      'http://buster-ussuri.debian.net/debian/dists/pubkey.gpg',
       'https://packages.treasuredata.com/GPG-KEY-td-agent',
       'https://repos.influxdata.com/influxdb.key',
-      'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key',
     ] %}
 {% endif %}
 
diff --git a/docker/base/apt_preferences.debian b/docker/base/apt_preferences.debian
index ef693a5d4..e3f7bb790 100644
--- a/docker/base/apt_preferences.debian
+++ b/docker/base/apt_preferences.debian
@@ -1,53 +1,8 @@
-# We want Ceph/nautilus
-Package: ceph* libceph* librados* librbd* librgw* python3-ceph* python3-rados python3-rbd python3-rgw radosgw
-Pin: version 14.*
-Pin-Priority: 1000
-
-# ceph-osd requires smartmontools from backports
-Package: smartmontools
-Pin: version 7.*
-Pin-Priority: 1000
-
 Package: rabbitmq-server
 Pin: version 3.8.*
 Pin-Priority: 1000
 
-# We need erlang 22+ for rabbitmq (aarch64 has v23, x86-64 has v22 and v23 so we use v23)
+# RabbitMQ wants 23+ and we want to stay on 23 if possible
 Package: erlang*
 Pin: version 1:23.*
 Pin-Priority: 1000
-
-# We do not want packages from this repo
-Package: *
-Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
-Pin-Priority: 100
-
-# We need libvirt 7.0.0 for nova
-Package: *libvirt*
-Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
-Pin-Priority: 600
-
-# on ppc64le we lack libvirt 7.0.0 so let fail with Debian one
-Package: *libvirt*
-Pin: release o=Debian,b=ppc64le
-Pin-Priority: 700
-
-# ovn-host depends on openvswitch from buster
-Package: openvswitch*
-Pin: release n=buster
-Pin-Priority: 700
-
-# we want newer QEMU
-Package: qemu*
-Pin: version 1:5.*
-Pin-Priority: 700
-
-# QEMU depends on those
-Package: libpmem1 libndctl6 libdaxctl1 pmdk-tools libpmemblk1 libpmemlog1 libpmemobj1 libpmempool1 liburing1 libfdt1
-Pin: release n=buster-backports
-Pin-Priority: 700
-
-# nova wants data in /usr/share/qemu/firmware
-Package: qemu-efi-aarch64
-Pin: release n=buster-backports
-Pin-Priority: 700
diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian
index d74f36c33..6b975b67c 100644
--- a/docker/base/sources.list.debian
+++ b/docker/base/sources.list.debian
@@ -1,33 +1,8 @@
 # Default repos
-deb http://deb.debian.org/debian buster main
+deb http://deb.debian.org/debian bullseye main
 
 # debian security updates
-deb http://security.debian.org/debian-security buster/updates main
+deb http://deb.debian.org/debian-security bullseye-security main
 
 # debian backports
-deb http://deb.debian.org/debian buster-backports main
-
-# Buster - Victoria repos
-deb http://buster-victoria.debian.net/debian buster-victoria-backports main
-deb http://buster-victoria.debian.net/debian buster-victoria-backports-nochange main
-
-# NOTE(hrw): repositories below are added into image when they are needed as
-# separate files in /etc/apt/sources.list.d/ directory. For that purpose they
-# are defined in kolla/repos.yaml file.
-#
-# Here they stay in case someone needs them.
-
-# elasticsearch (arch:all), logstash (arch:all), kibana (arch:amd64)
-#deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main
-
-# Grafana repo
-#deb https://packages.grafana.com/oss/deb stable main
-
-# td-agent for fluentd
-#deb http://packages.treasuredata.com/3/debian/buster buster contrib
-
-# rabbitmq repo
-#deb https://dl.bintray.com/rabbitmq/debian/ buster main
-
-# erlang repo
-#deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang
+deb http://deb.debian.org/debian bullseye-backports main
diff --git a/docker/mariadb/mariadb-server/Dockerfile.j2 b/docker/mariadb/mariadb-server/Dockerfile.j2
index b96e922fe..9ee7ef37d 100644
--- a/docker/mariadb/mariadb-server/Dockerfile.j2
+++ b/docker/mariadb/mariadb-server/Dockerfile.j2
@@ -37,7 +37,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 COPY mariadb_sudoers /etc/sudoers.d/kolla_mariadb_sudoers
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
+{% if base_distro == 'debian' %}
+COPY security_reset.expect.debian /usr/local/bin/kolla_security_reset
+{% else %}
 COPY security_reset.expect /usr/local/bin/kolla_security_reset
+{% endif %}
 RUN chmod 755 /usr/local/bin/kolla_extend_start \
     && chmod 755 /usr/local/bin/kolla_security_reset \
     && chmod 750 /etc/sudoers.d \
diff --git a/docker/mariadb/mariadb-server/security_reset.expect.debian b/docker/mariadb/mariadb-server/security_reset.expect.debian
new file mode 100644
index 000000000..ec40eaf90
--- /dev/null
+++ b/docker/mariadb/mariadb-server/security_reset.expect.debian
@@ -0,0 +1,67 @@
+#!/usr/bin/expect -f
+
+if [catch {set timeout $env(DB_MAX_TIMEOUT)}] {set timeout 10}
+spawn mysql_secure_installation
+expect {
+  timeout { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  "Enter current password for root (enter for none):"
+}
+send "\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Switch to unix_socket authentication [Y/n] ' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Switch to unix_socket authentication' prompt\n"; exit 1 }
+  "Switch to unix_socket authentication \\\[Y/n\\\] "
+}
+send "n\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Change the root password? [Y/n]' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Change the root password?' prompt\n"; exit 1 }
+  "Change the root password? \\\[Y/n\\\] "
+}
+send "y\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  "New password:"
+}
+send "$env(DB_ROOT_PASSWORD)\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  "Re-enter new password:"
+}
+send "$env(DB_ROOT_PASSWORD)\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  "Remove anonymous users?"
+}
+send "y\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  "Disallow root login remotely?"
+}
+send "n\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  "Remove test database and access to it?"
+}
+send "y\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  "Reload privilege tables now?"
+}
+send "y\r"
+expect eof
diff --git a/docker/openvswitch/openvswitch-base/Dockerfile.j2 b/docker/openvswitch/openvswitch-base/Dockerfile.j2
index 5bc40ec47..1ddf669e9 100644
--- a/docker/openvswitch/openvswitch-base/Dockerfile.j2
+++ b/docker/openvswitch/openvswitch-base/Dockerfile.j2
@@ -34,7 +34,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
 
-RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump \
+# NOTE(hrw): Upstream moved tcpdump to /usr/bin/ in 4.99 version (used in Debian)
+RUN if [ -e /usr/sbin/tcpdump ]; then mv /usr/sbin/tcpdump /usr/bin/tcpdump; fi \
     && touch /usr/local/bin/kolla_openvswitch_extend_start \
     && chmod 755 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_openvswitch_extend_start
 
diff --git a/kolla/common/config.py b/kolla/common/config.py
index 96701e72d..23e5ead40 100755
--- a/kolla/common/config.py
+++ b/kolla/common/config.py
@@ -24,13 +24,13 @@ BASE_ARCH = ['x86_64', 'ppc64le', 'aarch64']
 DEFAULT_BASE_TAGS = {
     'centos': {'name': 'quay.io/centos/centos', 'tag': 'stream8'},
     'rhel': {'name': 'registry.access.redhat.com/ubi8', 'tag': 'latest'},
-    'debian': {'name': 'debian', 'tag': '10'},
+    'debian': {'name': 'debian', 'tag': 'bullseye'},
     'ubuntu': {'name': 'ubuntu', 'tag': '20.04'},
 }
 DISTRO_RELEASE = {
     'centos': '8',
     'rhel': '8',
-    'debian': '10',
+    'debian': 'testing/unstable',
     'ubuntu': '20.04',
 }
 OPENSTACK_RELEASE = 'wallaby'
diff --git a/kolla/image/build.py b/kolla/image/build.py
index a148c980f..27006e1ac 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -132,7 +132,7 @@ UNBUILDABLE_IMAGES = {
         "bifrost-base",  # tries to install 'mysql-server' which is not in
                          # Debian 'buster'
         "monasca-grafana",  # FIXME(hrw): some ssl issues to fix
-        "ovsdpdk",
+        "ovn-base",      # needs more checking
         "qdrouterd",
     },
 
@@ -162,14 +162,10 @@ UNBUILDABLE_IMAGES = {
     },
 
     'debian+binary': {
-        "cloudkitty-base",
-        "ironic-neutron-agent",
-        "nova-serialproxy",
-        "senlin-conductor",  # no binary package
-        "senlin-health-manager",  # no binary package
-        "tacker-base",
-        "trove-base",          # package is not installable
-        "neutron-mlnx-agent",
+        "cloudkitty-base",       # no support in Dockerfile
+        "ironic-neutron-agent",  # no support in Dockerfile
+        "nova-serialproxy",      # no binary package
+        "tacker-base",           # no binary package
     },
 
     'ubuntu+binary': {
@@ -697,7 +693,7 @@ class KollaWorker(object):
         if self.base in rh_base:
             self.conf.distro_python_version = "3.6"
         elif self.base in ['debian']:
-            self.conf.distro_python_version = "3.7"
+            self.conf.distro_python_version = "3.9"
         elif self.base in ['ubuntu']:
             self.conf.distro_python_version = "3.8"
         else:
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index 05fb8bcaf..f522a9e85 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -63,7 +63,6 @@ debian:
   erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz:/kolla-wallaby/debian-buster ./"
   logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
@@ -76,7 +75,6 @@ debian-aarch64:
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
   logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz:/kolla-wallaby/debian-buster ./"
   rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
   td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
 
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index 8dd34f2fd..10d70ce5a 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -529,7 +529,7 @@ class KollaWorkerTest(base.TestCase):
         """check distro_python_version for Debian"""
         self.conf.set_override('base', 'debian')
         kolla = build.KollaWorker(self.conf)
-        self.assertEqual('3.7', kolla.distro_python_version)
+        self.assertEqual('3.9', kolla.distro_python_version)
 
     def test_build_distro_python_version_rhel80(self):
         """check distro_python_version for RHEL8.0"""
diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2
index d56c601e1..6be8b794b 100644
--- a/tests/templates/template_overrides.j2
+++ b/tests/templates/template_overrides.j2
@@ -18,7 +18,7 @@ RUN echo registry={{ nodepool_npmjs_proxy }} > /etc/npmrc \
 {% if base_distro in ['debian', 'ubuntu'] %}
 {% endraw %}
 
-# NOTE(hrw): Debian 'buster' and Ubuntu 18.04 refuse to use unsigned repos
+# NOTE(hrw): Debian since 'buster' and Ubuntu since 18.04 refuse to use unsigned repos
 RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated \
  && echo 'Acquire::AllowInsecureRepositories "true";' > /etc/apt/apt.conf.d/99allow-insecure-repos