From ffaf102fecc20510513de4898ce85b5b119e1795 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Tue, 6 Apr 2021 13:14:49 +0200 Subject: Debian/Ubuntu: handle APT keys in proper way apt-key is deprecated and in Debian 'bullseye' it complain when we use it the way we did. The right way is to store key files in separate directory and then, for each repo source file, point to proper key. And this is what this patch does. Both distros use the same keys now. More info: https://michael-prokop.at/blog/2021/02/16/how-to-properly-use-3rd-party-debian-repository-signing-keys-with-apt/ Change-Id: Ie19bf46236907d79cb42a5e94cd1067e952b8cbb --- kolla/template/methods.py | 12 ++- kolla/template/repos.yaml | 202 ++++++++++++++++++++++++++++++++++++-------- kolla/tests/test_methods.py | 40 +++++++-- 3 files changed, 210 insertions(+), 44 deletions(-) (limited to 'kolla') diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 69434ef50..9f15cc528 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -115,8 +115,16 @@ def handle_repos(context, reponames, mode): commands += ' %s %s' % (rpm_switch, repo_list[repo]) elif base_package_type == 'deb': if mode == 'enable': - commands += 'echo "%s" ' % repo_list[repo] - commands += '>/etc/apt/sources.list.d/%s.list && ' % repo + commands += f"""echo 'Uris: {repo_list[repo]['url']}' \ +>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Components: {repo_list[repo]['component']}' \ +>>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Suites: {repo_list[repo]['suite']}' \ +>>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Signed-By: /etc/kolla/apt-keys/{repo_list[repo]['gpg_key']}' \ +>>/etc/apt/sources.list.d/{repo}.sources \ +&& """ except KeyError: # NOTE(hrw): we ignore missing repositories for a given # distro/arch diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index d0ea99cbd..1d3fc3b17 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -37,45 +37,177 @@ centos-aarch64: rabbitmq: "rabbitmq_rabbitmq-server" td-agent: "treasuredata" -# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs debian: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/debian buster stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" - td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "bionic" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/debian" + suite: "buster" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + kibana: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/" + suite: "buster" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/debian/buster" + suite: "buster" + component: "contrib" + gpg_key: "treasuredata.asc" debian-aarch64: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/debian buster stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" - td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "bionic" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/debian" + suite: "buster" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + kibana: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/" + suite: "buster" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/debian/buster" + suite: "buster" + component: "contrib" + gpg_key: "treasuredata.asc" ubuntu: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" - qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" - td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "focal" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/ubuntu" + suite: "focal" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + kibana: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + mariadb: + url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu" + suite: "focal" + component: "main" + gpg_key: "mariadb.gpg" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/" + suite: "focal" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/ubuntu/focal/" + suite: "focal" + component: "contrib" + gpg_key: "treasuredata.asc" ubuntu-aarch64: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" - td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "focal" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/ubuntu" + suite: "focal" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + mariadb: + url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu" + suite: "focal" + component: "main" + gpg_key: "mariadb.gpg" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/" + suite: "focal" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/ubuntu/focal/" + suite: "focal" + component: "contrib" + gpg_key: "treasuredata.asc" diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py index 1ff8ec30f..81bb29946 100644 --- a/kolla/tests/test_methods.py +++ b/kolla/tests/test_methods.py @@ -66,8 +66,16 @@ class MethodsTest(base.TestCase): } result = methods.handle_repos(template_vars, ['grafana'], 'enable') - expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb ' - expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list' + expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' " + expectCmd += ">/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Components: main' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Types: deb' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Suites: stable' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources" self.assertEqual(expectCmd, result) def test_enable_repos_debian_missing_repo(self): @@ -91,11 +99,29 @@ class MethodsTest(base.TestCase): result = methods.handle_repos(template_vars, ['grafana', 'kibana'], 'enable') - expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb ' - expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && ' - expectCmd += 'echo "deb ' - expectCmd += 'https://artifacts.elastic.co/packages/oss-7.x/apt ' - expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list' + expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' " + expectCmd += ">/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Components: main' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Types: deb' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Suites: stable' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + + expectCmd += "echo 'Uris: " + expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' " + expectCmd += ">/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Components: main' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Types: deb' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Suites: stable' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources" + self.assertEqual(expectCmd, result) def test_disable_repos_centos(self): -- cgit v1.2.3