Debian/Ubuntu: handle APT keys in proper way

apt-key is deprecated and in Debian 'bullseye' it complain when we use
it the way we did.

The right way is to store key files in separate directory and then, for
each repo source file, point to proper key. And this is what this patch
does.

Both distros use the same keys now.

More info: https://michael-prokop.at/blog/2021/02/16/how-to-properly-use-3rd-party-debian-repository-signing-keys-with-apt/

Change-Id: Ie19bf46236907d79cb42a5e94cd1067e952b8cbb

5 files changed, 239 insertions, 90 deletions

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index 4d426c6c9..0e2e14bbb 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -252,13 +252,11 @@ ENV DEBIAN_FRONTEND noninteractive
 COPY dpkg_reducing_disk_footprint /etc/dpkg/dpkg.cfg.d/dpkg_reducing_disk_footprint
 
 {% block base_ubuntu_package_pre %}
-# Need apt-transport-https and ca-certificates before replacing sources.list or
-# apt-get update will not work if any repositories are accessed via HTTPS
+# curl and ca-certificates to fetch remote keys via http
+# gnupg to fetch keys directly from keyserver
 {% set base_ubuntu_package_pre_packages = [
-    'apt-transport-https',
     'ca-certificates',
     'curl',
-    'dirmngr',
     'gnupg'
 ] %}
 {{ macros.install_packages(base_ubuntu_package_pre_packages | customizable("base_ubuntu_package_pre_packages")) }}
@@ -294,7 +292,6 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
 
 {% set base_apt_packages = [
    'apt-utils',
-   'curl',
    'dumb-init',
    'gawk',
    'iproute2',
@@ -311,55 +308,29 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
    'tgt'
 ] %}
 
-{% if base_distro == 'ubuntu' %}
-    {# 391A9AA2147192839E9DB0315EDB1B62EC4926EA -- Canonical Cloud Archive Signing Key <ftpmaster@canonical.com> #}
-    {# 46095ACC8548582C1A2699A9D27D666CD88E42B4 -- Elasticsearch (Elasticsearch Signing Key) <dev_ops@elasticsearch.org> #}
-    {# 58118E89F3A912897C070ADBF76221572C52609D -- Docker Release Tool (releasedocker) <docker@docker.com> #}
-    {# 4D8EB5FDA37AB55F41A135203BF88A0C6A770882 -- Apache Qpid PPA Signing Key <dev@qpid.apache.org> #}
-    {# 901F9177AB97ACBE                         -- Treasure Data, Inc (Treasure Agent Official Signing key) <support@treasure-data.com> #}
-    {# A20F259AEB9C94BB                         -- Sensuapp (Freight) <support@hw-ops.com> #}
-    {# F1656F24C74CD1D8                         -- MariaDB Signing Key <signing-key@mariadb.org> #}
-    {# F77F1EDA57EBB1CC                         -- Launchpad RabbitMQ Erlang PPA key #}
-    {# F6609E60DC62814E                         -- PackageCloud RabbitMQ repository key #}
-    {% set base_apt_keys = [
-      '391A9AA2147192839E9DB0315EDB1B62EC4926EA',
-      '46095ACC8548582C1A2699A9D27D666CD88E42B4',
-      '49B07274951063870A8B7EAE7B8AA1A344C05248',
-      '58118E89F3A912897C070ADBF76221572C52609D',
-      '4D8EB5FDA37AB55F41A135203BF88A0C6A770882',
-      '901F9177AB97ACBE',
-      'A20F259AEB9C94BB',
-      'F1656F24C74CD1D8',
-      'F77F1EDA57EBB1CC',
-      'F6609E60DC62814E',
-    ] %}
-    {% set remote_apt_keys = [
-      'https://packages.grafana.com/gpg.key',
-      'https://repos.influxdata.com/influxdb.key',
-    ] %}
-{% elif base_distro == 'debian' %}
-    {% set base_apt_keys = [
-      '46095ACC8548582C1A2699A9D27D666CD88E42B4',
-      'F77F1EDA57EBB1CC',
-      'F6609E60DC62814E',
-    ] %}
-    {% set remote_apt_keys = [
-      'https://download.docker.com/linux/debian/gpg',
-      'https://packages.grafana.com/gpg.key',
-      'https://packages.treasuredata.com/GPG-KEY-td-agent',
-      'https://repos.influxdata.com/influxdb.key',
-    ] %}
-{% endif %}
+{% set base_apt_keys = [
+   {'name': 'erlang',   'keyid': 'F77F1EDA57EBB1CC'},
+   {'name': 'rabbitmq', 'keyid': 'F6609E60DC62814E'},
+] %}
+
+{# NOTE(hrw): type field defaults to 'asc' which is used for single keys #}
+{% set remote_apt_keys = [
+   {'name': 'elasticsearch', 'url': 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'},
+   {'name': 'grafana', 'url': 'https://packages.grafana.com/gpg.key'},
+   {'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdb.key'},
+   {'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'},
+   {'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'},
+] %}
 
 {% block base_ubuntu_package_installation %}
     {%- block base_ubuntu_package_key_installation %}
         {% for key in base_apt_keys | customizable('apt_keys') %}
-            {%- if loop.first %}RUN {% else %} && {% endif %}apt-key adv --no-tty --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 {{ key }}
+            {%- if loop.first %}RUN mkdir -p /etc/kolla/apt-keys/{% endif %} && gpg --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 {{ key.keyid }} && gpg --export {{ key.keyid }} >/etc/kolla/apt-keys/{{ key.name }}.gpg
             {%- if not loop.last %} \
             {% endif -%}
         {% endfor %}
         {% for key in remote_apt_keys | customizable('remote_apt_keys') %}
-            {%- if loop.first %} RUN {% else %} && {% endif %}curl {{ key }} | apt-key add -
+            {%- if loop.first %} RUN mkdir -p /etc/kolla/apt-keys/ {% endif %} && curl {{ key.url }} -o /etc/kolla/apt-keys/{{ key.name }}.{{ key.type | default('asc') }}
             {%- if not loop.last %} \
             {% endif -%}
         {% endfor %}
diff --git a/kolla/template/methods.py b/kolla/template/methods.py
index 69434ef50..9f15cc528 100644
--- a/kolla/template/methods.py
+++ b/kolla/template/methods.py
@@ -115,8 +115,16 @@ def handle_repos(context, reponames, mode):
                 commands += ' %s %s' % (rpm_switch, repo_list[repo])
             elif base_package_type == 'deb':
                 if mode == 'enable':
-                    commands += 'echo "%s" ' % repo_list[repo]
-                    commands += '>/etc/apt/sources.list.d/%s.list && ' % repo
+                    commands += f"""echo 'Uris: {repo_list[repo]['url']}' \
+>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Components: {repo_list[repo]['component']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Suites: {repo_list[repo]['suite']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Signed-By: /etc/kolla/apt-keys/{repo_list[repo]['gpg_key']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& """
         except KeyError:
             # NOTE(hrw): we ignore missing repositories for a given
             # distro/arch
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index d0ea99cbd..1d3fc3b17 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -37,45 +37,177 @@ centos-aarch64:
   rabbitmq: "rabbitmq_rabbitmq-server"
   td-agent: "treasuredata"
 
-# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
 debian:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "bionic"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/debian"
+    suite: "buster"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
+    suite: "buster"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/debian/buster"
+    suite: "buster"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 debian-aarch64:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "bionic"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/debian"
+    suite: "buster"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
+    suite: "buster"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/debian/buster"
+    suite: "buster"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 ubuntu:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
-  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/ubuntu"
+    suite: "focal"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  mariadb:
+    url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "mariadb.gpg"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
+    suite: "focal"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/ubuntu/focal/"
+    suite: "focal"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 ubuntu-aarch64:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
-  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/ubuntu"
+    suite: "focal"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  mariadb:
+    url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "mariadb.gpg"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
+    suite: "focal"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/ubuntu/focal/"
+    suite: "focal"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py
index 1ff8ec30f..81bb29946 100644
--- a/kolla/tests/test_methods.py
+++ b/kolla/tests/test_methods.py
@@ -66,8 +66,16 @@ class MethodsTest(base.TestCase):
         }
 
         result = methods.handle_repos(template_vars, ['grafana'], 'enable')
-        expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list'
+        expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
+        expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources"
         self.assertEqual(expectCmd, result)
 
     def test_enable_repos_debian_missing_repo(self):
@@ -91,11 +99,29 @@ class MethodsTest(base.TestCase):
 
         result = methods.handle_repos(template_vars, ['grafana', 'kibana'],
                                       'enable')
-        expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && '
-        expectCmd += 'echo "deb '
-        expectCmd += 'https://artifacts.elastic.co/packages/oss-7.x/apt '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list'
+        expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
+        expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+
+        expectCmd += "echo 'Uris: "
+        expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' "
+        expectCmd += ">/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources"
+
         self.assertEqual(expectCmd, result)
 
     def test_disable_repos_centos(self):
diff --git a/releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml b/releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml
new file mode 100644
index 000000000..8edc1a2f3
--- /dev/null
+++ b/releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml
@@ -0,0 +1,12 @@
+---
+upgrade:
+  - |
+    Format of APT keys has changed from simple list into dictionary. For
+    ``base_apt_keys`` we now use name and key ids and for ``remote_apt_keys``
+    names and URLs.
+
+    This allows to instruct APT to use those keys only for their repositories
+    instead of trusting them for all possible packages.
+
+    If you override ``base_apt_keys`` or ``remote_apt_keys`` then please adapt
+    to the new format.