diff options
| -rw-r--r-- | docker/base/Dockerfile.j2 | 63 | ||||
| -rw-r--r-- | kolla/template/methods.py | 12 | ||||
| -rw-r--r-- | kolla/template/repos.yaml | 202 | ||||
| -rw-r--r-- | kolla/tests/test_methods.py | 40 | ||||
| -rw-r--r-- | releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml | 12 |
5 files changed, 239 insertions, 90 deletions
diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 4d426c6c9..0e2e14bbb 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -252,13 +252,11 @@ ENV DEBIAN_FRONTEND noninteractive COPY dpkg_reducing_disk_footprint /etc/dpkg/dpkg.cfg.d/dpkg_reducing_disk_footprint {% block base_ubuntu_package_pre %} -# Need apt-transport-https and ca-certificates before replacing sources.list or -# apt-get update will not work if any repositories are accessed via HTTPS +# curl and ca-certificates to fetch remote keys via http +# gnupg to fetch keys directly from keyserver {% set base_ubuntu_package_pre_packages = [ - 'apt-transport-https', 'ca-certificates', 'curl', - 'dirmngr', 'gnupg' ] %} {{ macros.install_packages(base_ubuntu_package_pre_packages | customizable("base_ubuntu_package_pre_packages")) }} @@ -294,7 +292,6 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {% set base_apt_packages = [ 'apt-utils', - 'curl', 'dumb-init', 'gawk', 'iproute2', @@ -311,55 +308,29 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom 'tgt' ] %} -{% if base_distro == 'ubuntu' %} - {# 391A9AA2147192839E9DB0315EDB1B62EC4926EA -- Canonical Cloud Archive Signing Key <ftpmaster@canonical.com> #} - {# 46095ACC8548582C1A2699A9D27D666CD88E42B4 -- Elasticsearch (Elasticsearch Signing Key) <dev_ops@elasticsearch.org> #} - {# 58118E89F3A912897C070ADBF76221572C52609D -- Docker Release Tool (releasedocker) <docker@docker.com> #} - {# 4D8EB5FDA37AB55F41A135203BF88A0C6A770882 -- Apache Qpid PPA Signing Key <dev@qpid.apache.org> #} - {# 901F9177AB97ACBE -- Treasure Data, Inc (Treasure Agent Official Signing key) <support@treasure-data.com> #} - {# A20F259AEB9C94BB -- Sensuapp (Freight) <support@hw-ops.com> #} - {# F1656F24C74CD1D8 -- MariaDB Signing Key <signing-key@mariadb.org> #} - {# F77F1EDA57EBB1CC -- Launchpad RabbitMQ Erlang PPA key #} - {# F6609E60DC62814E -- PackageCloud RabbitMQ repository key #} - {% set base_apt_keys = [ - '391A9AA2147192839E9DB0315EDB1B62EC4926EA', - '46095ACC8548582C1A2699A9D27D666CD88E42B4', - '49B07274951063870A8B7EAE7B8AA1A344C05248', - '58118E89F3A912897C070ADBF76221572C52609D', - '4D8EB5FDA37AB55F41A135203BF88A0C6A770882', - '901F9177AB97ACBE', - 'A20F259AEB9C94BB', - 'F1656F24C74CD1D8', - 'F77F1EDA57EBB1CC', - 'F6609E60DC62814E', - ] %} - {% set remote_apt_keys = [ - 'https://packages.grafana.com/gpg.key', - 'https://repos.influxdata.com/influxdb.key', - ] %} -{% elif base_distro == 'debian' %} - {% set base_apt_keys = [ - '46095ACC8548582C1A2699A9D27D666CD88E42B4', - 'F77F1EDA57EBB1CC', - 'F6609E60DC62814E', - ] %} - {% set remote_apt_keys = [ - 'https://download.docker.com/linux/debian/gpg', - 'https://packages.grafana.com/gpg.key', - 'https://packages.treasuredata.com/GPG-KEY-td-agent', - 'https://repos.influxdata.com/influxdb.key', - ] %} -{% endif %} +{% set base_apt_keys = [ + {'name': 'erlang', 'keyid': 'F77F1EDA57EBB1CC'}, + {'name': 'rabbitmq', 'keyid': 'F6609E60DC62814E'}, +] %} + +{# NOTE(hrw): type field defaults to 'asc' which is used for single keys #} +{% set remote_apt_keys = [ + {'name': 'elasticsearch', 'url': 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'}, + {'name': 'grafana', 'url': 'https://packages.grafana.com/gpg.key'}, + {'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdb.key'}, + {'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'}, + {'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'}, +] %} {% block base_ubuntu_package_installation %} {%- block base_ubuntu_package_key_installation %} {% for key in base_apt_keys | customizable('apt_keys') %} - {%- if loop.first %}RUN {% else %} && {% endif %}apt-key adv --no-tty --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 {{ key }} + {%- if loop.first %}RUN mkdir -p /etc/kolla/apt-keys/{% endif %} && gpg --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 {{ key.keyid }} && gpg --export {{ key.keyid }} >/etc/kolla/apt-keys/{{ key.name }}.gpg {%- if not loop.last %} \ {% endif -%} {% endfor %} {% for key in remote_apt_keys | customizable('remote_apt_keys') %} - {%- if loop.first %} RUN {% else %} && {% endif %}curl {{ key }} | apt-key add - + {%- if loop.first %} RUN mkdir -p /etc/kolla/apt-keys/ {% endif %} && curl {{ key.url }} -o /etc/kolla/apt-keys/{{ key.name }}.{{ key.type | default('asc') }} {%- if not loop.last %} \ {% endif -%} {% endfor %} diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 19fcf4e22..168f30451 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -115,8 +115,16 @@ def handle_repos(context, reponames, mode): commands += ' %s %s' % (rpm_switch, repo_list[repo]) elif base_package_type == 'deb': if mode == 'enable': - commands += 'echo "%s" ' % repo_list[repo] - commands += '>/etc/apt/sources.list.d/%s.list && ' % repo + commands += f"""echo 'Uris: {repo_list[repo]['url']}' \ +>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Components: {repo_list[repo]['component']}' \ +>>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Suites: {repo_list[repo]['suite']}' \ +>>/etc/apt/sources.list.d/{repo}.sources \ +&& echo 'Signed-By: /etc/kolla/apt-keys/{repo_list[repo]['gpg_key']}' \ +>>/etc/apt/sources.list.d/{repo}.sources \ +&& """ except KeyError: # NOTE(hrw): we ignore missing repositories for a given # distro/arch diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index d0ea99cbd..1d3fc3b17 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -37,45 +37,177 @@ centos-aarch64: rabbitmq: "rabbitmq_rabbitmq-server" td-agent: "treasuredata" -# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs debian: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/debian buster stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" - td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "bionic" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/debian" + suite: "buster" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + kibana: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/" + suite: "buster" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/debian/buster" + suite: "buster" + component: "contrib" + gpg_key: "treasuredata.asc" debian-aarch64: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/debian buster stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" - td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "bionic" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/debian" + suite: "buster" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + kibana: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/" + suite: "buster" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/debian/buster" + suite: "buster" + component: "contrib" + gpg_key: "treasuredata.asc" ubuntu: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" - qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" - td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "focal" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/ubuntu" + suite: "focal" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + kibana: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + mariadb: + url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu" + suite: "focal" + component: "main" + gpg_key: "mariadb.gpg" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/" + suite: "focal" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/ubuntu/focal/" + suite: "focal" + component: "contrib" + gpg_key: "treasuredata.asc" ubuntu-aarch64: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" - grafana: "deb https://packages.grafana.com/oss/deb stable main" - influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" - logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" - td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" + elasticsearch: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + erlang: + url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "focal" + component: "main" + gpg_key: "erlang.gpg" + grafana: + url: "https://packages.grafana.com/oss/deb" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/ubuntu" + suite: "focal" + component: "stable" + gpg_key: "influxdb.asc" + logstash: + url: "https://artifacts.elastic.co/packages/oss-7.x/apt" + suite: "stable" + component: "main" + gpg_key: "elasticsearch.asc" + mariadb: + url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu" + suite: "focal" + component: "main" + gpg_key: "mariadb.gpg" + rabbitmq: + url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/" + suite: "focal" + component: "main" + gpg_key: "rabbitmq.gpg" + td-agent: + url: "http://packages.treasuredata.com/4/ubuntu/focal/" + suite: "focal" + component: "contrib" + gpg_key: "treasuredata.asc" diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py index 1ff8ec30f..81bb29946 100644 --- a/kolla/tests/test_methods.py +++ b/kolla/tests/test_methods.py @@ -66,8 +66,16 @@ class MethodsTest(base.TestCase): } result = methods.handle_repos(template_vars, ['grafana'], 'enable') - expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb ' - expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list' + expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' " + expectCmd += ">/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Components: main' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Types: deb' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Suites: stable' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources" self.assertEqual(expectCmd, result) def test_enable_repos_debian_missing_repo(self): @@ -91,11 +99,29 @@ class MethodsTest(base.TestCase): result = methods.handle_repos(template_vars, ['grafana', 'kibana'], 'enable') - expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb ' - expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && ' - expectCmd += 'echo "deb ' - expectCmd += 'https://artifacts.elastic.co/packages/oss-7.x/apt ' - expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list' + expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' " + expectCmd += ">/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Components: main' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Types: deb' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Suites: stable' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' " + expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && " + + expectCmd += "echo 'Uris: " + expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' " + expectCmd += ">/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Components: main' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Types: deb' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Suites: stable' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && " + expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' " + expectCmd += ">>/etc/apt/sources.list.d/kibana.sources" + self.assertEqual(expectCmd, result) def test_disable_repos_centos(self): diff --git a/releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml b/releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml new file mode 100644 index 000000000..8edc1a2f3 --- /dev/null +++ b/releasenotes/notes/apt-keys-bb7c8678fc6d908b.yaml @@ -0,0 +1,12 @@ +--- +upgrade: + - | + Format of APT keys has changed from simple list into dictionary. For + ``base_apt_keys`` we now use name and key ids and for ``remote_apt_keys`` + names and URLs. + + This allows to instruct APT to use those keys only for their repositories + instead of trusting them for all possible packages. + + If you override ``base_apt_keys`` or ``remote_apt_keys`` then please adapt + to the new format. |