Debian/Ubuntu: handle APT keys in proper way

apt-key is deprecated and in Debian 'bullseye' it complain when we use
it the way we did.

The right way is to store key files in separate directory and then, for
each repo source file, point to proper key. And this is what this patch
does.

Both distros use the same keys now.

More info: https://michael-prokop.at/blog/2021/02/16/how-to-properly-use-3rd-party-debian-repository-signing-keys-with-apt/

Change-Id: Ie19bf46236907d79cb42a5e94cd1067e952b8cbb

3 files changed, 210 insertions, 44 deletions

diff --git a/kolla/template/methods.py b/kolla/template/methods.py
index 69434ef50..9f15cc528 100644
--- a/kolla/template/methods.py
+++ b/kolla/template/methods.py
@@ -115,8 +115,16 @@ def handle_repos(context, reponames, mode):
                 commands += ' %s %s' % (rpm_switch, repo_list[repo])
             elif base_package_type == 'deb':
                 if mode == 'enable':
-                    commands += 'echo "%s" ' % repo_list[repo]
-                    commands += '>/etc/apt/sources.list.d/%s.list && ' % repo
+                    commands += f"""echo 'Uris: {repo_list[repo]['url']}' \
+>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Components: {repo_list[repo]['component']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Suites: {repo_list[repo]['suite']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Signed-By: /etc/kolla/apt-keys/{repo_list[repo]['gpg_key']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& """
         except KeyError:
             # NOTE(hrw): we ignore missing repositories for a given
             # distro/arch
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index d0ea99cbd..1d3fc3b17 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -37,45 +37,177 @@ centos-aarch64:
   rabbitmq: "rabbitmq_rabbitmq-server"
   td-agent: "treasuredata"
 
-# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
 debian:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "bionic"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/debian"
+    suite: "buster"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
+    suite: "buster"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/debian/buster"
+    suite: "buster"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 debian-aarch64:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "bionic"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/debian"
+    suite: "buster"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
+    suite: "buster"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/debian/buster"
+    suite: "buster"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 ubuntu:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
-  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/ubuntu"
+    suite: "focal"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  mariadb:
+    url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "mariadb.gpg"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
+    suite: "focal"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/ubuntu/focal/"
+    suite: "focal"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 ubuntu-aarch64:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
-  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/ubuntu"
+    suite: "focal"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  mariadb:
+    url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "mariadb.gpg"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
+    suite: "focal"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/ubuntu/focal/"
+    suite: "focal"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py
index 1ff8ec30f..81bb29946 100644
--- a/kolla/tests/test_methods.py
+++ b/kolla/tests/test_methods.py
@@ -66,8 +66,16 @@ class MethodsTest(base.TestCase):
         }
 
         result = methods.handle_repos(template_vars, ['grafana'], 'enable')
-        expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list'
+        expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
+        expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources"
         self.assertEqual(expectCmd, result)
 
     def test_enable_repos_debian_missing_repo(self):
@@ -91,11 +99,29 @@ class MethodsTest(base.TestCase):
 
         result = methods.handle_repos(template_vars, ['grafana', 'kibana'],
                                       'enable')
-        expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && '
-        expectCmd += 'echo "deb '
-        expectCmd += 'https://artifacts.elastic.co/packages/oss-7.x/apt '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list'
+        expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
+        expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+
+        expectCmd += "echo 'Uris: "
+        expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' "
+        expectCmd += ">/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources"
+
         self.assertEqual(expectCmd, result)
 
     def test_disable_repos_centos(self):