Merge "Debian/Ubuntu: handle APT keys in proper way"

3 files changed, 210 insertions, 44 deletions

diff --git a/kolla/template/methods.py b/kolla/template/methods.py
index 19fcf4e22..168f30451 100644
--- a/kolla/template/methods.py
+++ b/kolla/template/methods.py
@@ -115,8 +115,16 @@ def handle_repos(context, reponames, mode):
                 commands += ' %s %s' % (rpm_switch, repo_list[repo])
             elif base_package_type == 'deb':
                 if mode == 'enable':
-                    commands += 'echo "%s" ' % repo_list[repo]
-                    commands += '>/etc/apt/sources.list.d/%s.list && ' % repo
+                    commands += f"""echo 'Uris: {repo_list[repo]['url']}' \
+>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Components: {repo_list[repo]['component']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Suites: {repo_list[repo]['suite']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& echo 'Signed-By: /etc/kolla/apt-keys/{repo_list[repo]['gpg_key']}' \
+>>/etc/apt/sources.list.d/{repo}.sources \
+&& """
         except KeyError:
             # NOTE(hrw): we ignore missing repositories for a given
             # distro/arch
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index d0ea99cbd..1d3fc3b17 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -37,45 +37,177 @@ centos-aarch64:
   rabbitmq: "rabbitmq_rabbitmq-server"
   td-agent: "treasuredata"
 
-# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
 debian:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "bionic"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/debian"
+    suite: "buster"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
+    suite: "buster"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/debian/buster"
+    suite: "buster"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 debian-aarch64:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "bionic"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/debian"
+    suite: "buster"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
+    suite: "buster"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/debian/buster"
+    suite: "buster"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 ubuntu:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
-  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/ubuntu"
+    suite: "focal"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  kibana:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  mariadb:
+    url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "mariadb.gpg"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
+    suite: "focal"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/ubuntu/focal/"
+    suite: "focal"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
 
 ubuntu-aarch64:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
-  grafana: "deb https://packages.grafana.com/oss/deb stable main"
-  influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
-  logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
-  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
+  elasticsearch:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  erlang:
+    url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "erlang.gpg"
+  grafana:
+    url: "https://packages.grafana.com/oss/deb"
+    suite: "stable"
+    component: "main"
+    gpg_key: "grafana.asc"
+  influxdb:
+    url: "https://repos.influxdata.com/ubuntu"
+    suite: "focal"
+    component: "stable"
+    gpg_key: "influxdb.asc"
+  logstash:
+    url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
+    suite: "stable"
+    component: "main"
+    gpg_key: "elasticsearch.asc"
+  mariadb:
+    url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
+    suite: "focal"
+    component: "main"
+    gpg_key: "mariadb.gpg"
+  rabbitmq:
+    url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
+    suite: "focal"
+    component: "main"
+    gpg_key: "rabbitmq.gpg"
+  td-agent:
+    url: "http://packages.treasuredata.com/4/ubuntu/focal/"
+    suite: "focal"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py
index 1ff8ec30f..81bb29946 100644
--- a/kolla/tests/test_methods.py
+++ b/kolla/tests/test_methods.py
@@ -66,8 +66,16 @@ class MethodsTest(base.TestCase):
         }
 
         result = methods.handle_repos(template_vars, ['grafana'], 'enable')
-        expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list'
+        expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
+        expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources"
         self.assertEqual(expectCmd, result)
 
     def test_enable_repos_debian_missing_repo(self):
@@ -91,11 +99,29 @@ class MethodsTest(base.TestCase):
 
         result = methods.handle_repos(template_vars, ['grafana', 'kibana'],
                                       'enable')
-        expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && '
-        expectCmd += 'echo "deb '
-        expectCmd += 'https://artifacts.elastic.co/packages/oss-7.x/apt '
-        expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list'
+        expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
+        expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
+
+        expectCmd += "echo 'Uris: "
+        expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' "
+        expectCmd += ">/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Components: main' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Types: deb' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Suites: stable' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
+        expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' "
+        expectCmd += ">>/etc/apt/sources.list.d/kibana.sources"
+
         self.assertEqual(expectCmd, result)
 
     def test_disable_repos_centos(self):