aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/source/admin/image-building.rst91
-rw-r--r--kolla/common/config.py3
-rw-r--r--kolla/docker/base/Dockerfile.j214
-rw-r--r--kolla/docker/base/ubuntu.sources7
-rw-r--r--kolla/template/methods.py105
-rw-r--r--kolla/template/repos.yaml44
-rw-r--r--kolla/tests/test_methods.py212
-rw-r--r--releasenotes/notes/base-repos-in-yaml-f6a4a871da92c823.yaml23
-rw-r--r--roles/kolla-build/defaults/main.yml1
-rw-r--r--roles/kolla-build/tasks/main.yml9
-rw-r--r--tests/playbooks/run.yml10
-rw-r--r--tests/templates/repos.yaml.j263
-rw-r--r--tests/templates/template_overrides.j244
13 files changed, 532 insertions, 94 deletions
diff --git a/doc/source/admin/image-building.rst b/doc/source/admin/image-building.rst
index af59e4a2e..4a2c6773b 100644
--- a/doc/source/admin/image-building.rst
+++ b/doc/source/admin/image-building.rst
@@ -679,6 +679,97 @@ Can I use the ``--template-override`` option for custom templates? Yes!
Custom repos
------------
+repos.yaml
+^^^^^^^^^^
+
+Kolla uses ``kolla/template/repos.yaml`` to define repositories used during
+image builds. Each entry is keyed by a logical name and contains the
+information needed to configure the repository.
+
+Repositories marked with ``distro: True`` are default repos that require no
+additional configuration. For RPM-based distros they are enabled via
+``dnf config-manager --enable``; for Debian-based distros they are a no-op
+as the sources are already provided by the base image. Examples include
+``crb``, ``extras``, and the base repos ``baseos`` and ``appstream`` for
+CentOS and Rocky Linux, as well as ``debian``, ``debian-security``,
+``ubuntu``, ``ubuntu-security``.
+
+Repositories without ``distro: True`` are fully templated into a new
+repository file and must provide at least one of ``baseurl``, ``metalink``,
+or ``mirrorlist`` (RPM), or ``url`` (Debian/Ubuntu).
+
+To override repositories (e.g. to point base repos at a local mirror),
+provide a custom ``repos.yaml`` via the ``--repos-yaml`` option and supply
+entries without ``distro: True`` and with the desired URL.
+
+Example for Rocky Linux:
+
+.. code-block:: yaml
+
+ rocky:
+ baseos:
+ name: "baseos"
+ baseurl: "https://my-mirror.example.com/rocky/10/BaseOS/$basearch/os/"
+ gpgkey: "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10"
+ appstream:
+ name: "appstream"
+ baseurl: "https://my-mirror.example.com/rocky/10/AppStream/$basearch/os/"
+ gpgkey: "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10"
+ crb:
+ name: "crb"
+ baseurl: "https://my-mirror.example.com/rocky/10/CRB/$basearch/os/"
+ gpgkey: "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10"
+
+Example for Debian:
+
+.. code-block:: yaml
+
+ debian:
+ debian:
+ component: "main"
+ gpg_key: "/usr/share/keyrings/debian-archive-keyring.gpg"
+ suite: "trixie trixie-updates trixie-backports"
+ url: "https://my-mirror.example.com/debian"
+ debian-security:
+ component: "main"
+ gpg_key: "/usr/share/keyrings/debian-archive-keyring.gpg"
+ suite: "trixie-security"
+ url: "https://my-mirror.example.com/debian-security"
+
+Example for Ubuntu:
+
+.. code-block:: yaml
+
+ ubuntu:
+ ubuntu:
+ component: "main universe"
+ gpg_key: "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+ suite: "noble noble-updates noble-backports"
+ url: "https://my-mirror.example.com/ubuntu/"
+ ubuntu-security:
+ component: "main universe"
+ gpg_key: "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+ suite: "noble-security"
+ url: "https://my-mirror.example.com/ubuntu/"
+
+.. code-block:: ini
+
+ repos_yaml = /path/to/custom-repos.yaml
+
+When overriding a base repo, any existing ``.repo`` file on the image
+containing that repository ID is automatically removed before the new one
+is created.
+
+Some distro repos share a single ``.repo`` file on disk (for example,
+``baseos``, ``appstream``, and ``crb`` all live in ``rocky.repo`` on Rocky
+Linux). These repos are annotated with a ``file_group`` field in
+``repos.yaml`` that names the shared file. If any repo in a ``file_group``
+is overridden, all other repos sharing that ``file_group`` must also be
+overridden — otherwise removing the shared file would silently disable them.
+Kolla validates this at build time and raises an error listing the missing
+overrides. For Rocky Linux this means overriding ``baseos`` or ``appstream``
+requires also overriding ``crb``.
+
Red Hat
^^^^^^^
diff --git a/kolla/common/config.py b/kolla/common/config.py
index d46de4095..245afce50 100644
--- a/kolla/common/config.py
+++ b/kolla/common/config.py
@@ -246,7 +246,8 @@ _CLI_OPTS = [
'kolla-toolbox image'),
cfg.StrOpt('openstack-release-codename',
default=OPENSTACK_RELEASE_CODENAME,
- help='OpenStack release codename for Debian OpenStack extrepo'),
+ help='OpenStack release codename for Debian OpenStack extrepo '
+ 'and Ubuntu Cloud Archive'),
cfg.StrOpt('openstack-branch',
help='Branch for source images (internal; with a dash; '
'please set openstack-release instead)'),
diff --git a/kolla/docker/base/Dockerfile.j2 b/kolla/docker/base/Dockerfile.j2
index 0e2f3c1ab..6c1022ece 100644
--- a/kolla/docker/base/Dockerfile.j2
+++ b/kolla/docker/base/Dockerfile.j2
@@ -34,10 +34,8 @@ RUN cat /tmp/kolla_bashrc >> /etc/skel/.bashrc \
ENV PS1="$(tput bold)($(printenv KOLLA_SERVICE_NAME))$(tput sgr0)[$(id -un)@$(hostname -s) $(pwd)]$ "
{% if base_package_type == 'rpm' %}
-# For RPM Variants, enable the correct repositories - this should all be done
-# in the base image so repos are consistent throughout the system. This also
-# enables to provide repo overrides at a later date in a simple fashion if we
-# desire such functionality. I think we will :)
+
+{{ macros.enable_extra_repos(['baseos', 'appstream']) }}
ENV KOLLA_RPM_OVS_VERSION=3.5 \
KOLLA_RPM_OVN_VERSION=26.03
@@ -154,6 +152,7 @@ RUN {{ macros.install_packages(base_centos_yum_repo_packages | customizable("cen
{# endif for base_package_type rpm #}
{% elif base_package_type == 'deb' %}
+
# This will prevent questions from being asked during the install
ENV DEBIAN_FRONTEND noninteractive
@@ -195,9 +194,16 @@ COPY {{ base_distro }}.sources.arm64 /etc/apt/sources.list.d/{{ base_distro }}.s
{% else %}
COPY {{ base_distro }}.sources /etc/apt/sources.list.d/
{% endif %}
+
COPY sources.list /etc/apt/sources.list.d/kolla-custom.list
{% endblock %}
+{% if base_distro == 'debian' %}
+{{ macros.enable_extra_repos(['debian', 'debian-security']) }}
+{% elif base_distro == 'ubuntu' %}
+{{ macros.enable_extra_repos(['ubuntu', 'ubuntu-security', 'ubuntu-cloud-archive']) }}
+{% endif %}
+
{% block base_debian_after_sources_list %}{% endblock %}
{# install Debian Openstack repos - they are not mirrored on CI #}
diff --git a/kolla/docker/base/ubuntu.sources b/kolla/docker/base/ubuntu.sources
index f8f609cee..46b817b7e 100644
--- a/kolla/docker/base/ubuntu.sources
+++ b/kolla/docker/base/ubuntu.sources
@@ -12,13 +12,6 @@ Suites: noble-security
Components: main universe
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
-## Ubuntu Cloud Archive
-Types: deb
-URIs: http://ubuntu-cloud.archive.canonical.com/ubuntu
-Suites: noble-updates/gazpacho
-Components: main
-Signed-By: /usr/share/keyrings/ubuntu-cloud-keyring.gpg
-
# NOTE(hrw): extra repositories are added into image when they are needed as
# separate files in /etc/apt/sources.list.d/ directory. For that purpose they
# are defined in kolla/template/repos.yaml file.
diff --git a/kolla/template/methods.py b/kolla/template/methods.py
index 5dbf8da1b..3d6767206 100644
--- a/kolla/template/methods.py
+++ b/kolla/template/methods.py
@@ -24,7 +24,9 @@ APT_REPO = "echo 'Uris: {url}' >/etc/apt/sources.list.d/{repo}.sources && \
echo 'Components: {component}' >>/etc/apt/sources.list.d/{repo}.sources && \
echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources && \
echo 'Suites: {suite}' >>/etc/apt/sources.list.d/{repo}.sources && \
-echo 'Signed-By: /etc/kolla/apt-keys/{gpg_key}' \
+echo 'Signed-By: {signed_by}' \
+>>/etc/apt/sources.list.d/{repo}.sources"
+APT_TRUSTED = " && echo 'Trusted: yes' \
>>/etc/apt/sources.list.d/{repo}.sources"
DNF_BASEURL = " && echo 'baseurl={baseurl}' >>/etc/yum.repos.d/{repo}.repo"
DNF_DISABLE = "dnf config-manager --disable {name} || true"
@@ -35,6 +37,8 @@ DNF_GPGKEY_ADD = " && echo ' {gpgkey}' >>/etc/yum.repos.d/{repo}.repo"
DNF_METALINK = " && echo 'metalink={metalink}' >>/etc/yum.repos.d/{repo}.repo"
DNF_MIRRORLIST = " && \
echo 'mirrorlist={mirrorlist}' >>/etc/yum.repos.d/{repo}.repo"
+DNF_REMOVE_EXISTING = \
+ "grep -rlF '[{name}]' /etc/yum.repos.d/ 2>/dev/null | xargs -r rm -f"
DNF_REPO = "echo '[{name}]' >/etc/yum.repos.d/{repo}.repo && \
echo 'name={name}' >>/etc/yum.repos.d/{repo}.repo && \
echo 'enabled=1' >>/etc/yum.repos.d/{repo}.repo"
@@ -112,6 +116,7 @@ def handle_repos(context, reponames, mode):
os.path.realpath(__file__)) + '/repos.yaml'
with open(default_repofile, 'r') as repos_file:
repo_data = yaml.safe_load(repos_file)
+ default_repo_data = {k: dict(v) for k, v in repo_data.items()}
if context.get('repos_yaml'):
with open(context.get('repos_yaml'), 'r') as repos_file:
@@ -125,23 +130,63 @@ def handle_repos(context, reponames, mode):
base_distro = context.get('base_distro')
base_arch = context.get('base_arch')
image_name = context.get('image_name')
+ openstack_release_codename = context.get('openstack_release_codename')
commands = ''
+ def _build_repo_list(data):
+ result = {}
+ for section in (base_package_type, base_distro,
+ '%s-%s' % (base_distro, base_arch)):
+ for repo_name, repo_info in data.get(section, {}).items():
+ if repo_name in result:
+ merged = {**result[repo_name], **repo_info}
+ if any(k in merged for k in
+ ('baseurl', 'metalink', 'mirrorlist', 'url')):
+ merged.pop('distro', None)
+ result[repo_name] = merged
+ else:
+ result[repo_name] = repo_info
+ return result
+
try:
- repo_list = repo_data.get(base_package_type, dict()) | \
- repo_data.get(base_distro, dict()) | \
- repo_data.get('%s-%s' % (base_distro, base_arch), dict())
+ repo_list = _build_repo_list(repo_data)
except KeyError:
# NOTE(hrw): Fallback to distro list
repo_list = repo_data[base_distro]
+ if base_package_type == 'rpm' and context.get('repos_yaml'):
+ default_repo_list = _build_repo_list(default_repo_data)
+ distro_overridden = {r for r, d in repo_list.items()
+ if not d.get('distro')
+ and default_repo_list.get(r, {}).get('distro')}
+ if distro_overridden:
+ overridden_groups = {
+ default_repo_list[r].get('file_group')
+ for r in distro_overridden
+ if default_repo_list.get(r, {}).get('file_group')}
+ distro_not_overridden = sorted(
+ r for r, d in repo_list.items()
+ if d.get('distro')
+ and r not in distro_overridden
+ and d.get('file_group') in overridden_groups)
+ if distro_not_overridden:
+ raise ValueError(
+ "Repositories %s override distro-provided repos and will "
+ "remove their .repo file. Repositories %s are still using "
+ "distro defaults and share the same file. Please also "
+ "override them in your repos.yaml."
+ % (sorted(distro_overridden), distro_not_overridden))
+
for index, repo in enumerate(reponames):
try:
_repo = repo_list[repo]
if base_package_type == 'rpm':
if mode == 'enable':
if not _repo.get('distro'):
+ commands += DNF_REMOVE_EXISTING.format(
+ name=_repo['name'])
+ commands += " && "
commands += DNF_REPO.format(
name=_repo['name'],
repo=repo,
@@ -156,6 +201,14 @@ def handle_repos(context, reponames, mode):
repo_gpgcheck=_repo['repo_gpgcheck'],
repo=repo)
+ if not any(k in _repo for k in
+ ('baseurl', 'metalink', 'mirrorlist')):
+ raise ValueError(
+ "Repository '%s' has no baseurl, metalink,"
+ " or mirrorlist" % repo)
+ if 'gpgkey' not in _repo:
+ raise ValueError(
+ "Repository '%s' has no gpgkey" % repo)
# NOTE(mnasiadka): Support multiple gpgkeys
gpgkeys = _repo['gpgkey'].splitlines()
for _, gpgkey in enumerate(gpgkeys):
@@ -166,25 +219,23 @@ def handle_repos(context, reponames, mode):
commands += DNF_GPGKEY_ADD.format(
gpgkey=gpgkey,
repo=repo)
+ if 'baseurl' in _repo:
+ # NOTE(mnasiadka): Support multiple baseurls
+ baseurl = _repo['baseurl'].splitlines()
+ for url in baseurl:
+ commands += DNF_BASEURL.format(baseurl=url,
+ repo=repo)
+ elif 'metalink' in _repo:
+ commands += DNF_METALINK.format(
+ metalink=_repo['metalink'], repo=repo
+ )
+ elif 'mirrorlist' in _repo:
+ commands += DNF_MIRRORLIST.format(
+ mirrorlist=_repo['mirrorlist'], repo=repo
+ )
else:
commands += DNF_ENABLE.format(name=_repo['name'])
- if 'baseurl' in _repo:
- # NOTE(mnasiadka): Support multiple baseurls
- baseurl = _repo['baseurl'].splitlines()
- for url in baseurl:
- commands += DNF_BASEURL.format(baseurl=url,
- repo=repo)
- elif 'metalink' in _repo:
- commands += DNF_METALINK.format(
- metalink=_repo['metalink'], repo=repo
- )
-
- elif 'mirrorlist' in _repo:
- commands += DNF_MIRRORLIST.format(
- mirrorlist=_repo['mirrorlist'], repo=repo
- )
-
if index != len(reponames) - 1:
commands += " && "
@@ -192,14 +243,22 @@ def handle_repos(context, reponames, mode):
commands += DNF_DISABLE.format(name=_repo['name'])
elif base_package_type == "deb":
- if mode == "enable":
+ if mode == "enable" and not _repo.get('distro'):
+ gpg_key = _repo['gpg_key']
+ signed_by = gpg_key if gpg_key.startswith('/') \
+ else '/etc/kolla/apt-keys/' + gpg_key
+ suite = _repo['suite'].replace(
+ '{openstack_release_codename}',
+ openstack_release_codename.lower())
commands += APT_REPO.format(
component=_repo['component'],
- gpg_key=_repo['gpg_key'],
- suite=_repo['suite'],
+ signed_by=signed_by,
+ suite=suite,
url=_repo['url'],
repo=repo,
)
+ if _repo.get('trusted'):
+ commands += APT_TRUSTED.format(repo=repo)
if 'arch' in _repo:
commands += APT_ARCH.format(
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
index 508d82990..42e0185c4 100644
--- a/kolla/template/repos.yaml
+++ b/kolla/template/repos.yaml
@@ -20,6 +20,10 @@ deb:
gpg_key: "opensearch.asc"
debian:
+ debian:
+ distro: True
+ debian-security:
+ distro: True
erlang:
url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "noble"
@@ -63,6 +67,15 @@ debian-aarch64:
gpg_key: "rabbitmq.gpg"
ubuntu:
+ ubuntu:
+ distro: True
+ ubuntu-security:
+ distro: True
+ ubuntu-cloud-archive:
+ url: "http://ubuntu-cloud.archive.canonical.com/ubuntu"
+ suite: "noble-updates/{openstack_release_codename}"
+ component: "main"
+ gpg_key: "/usr/share/keyrings/ubuntu-cloud-keyring.gpg"
erlang:
url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "noble"
@@ -95,6 +108,10 @@ ubuntu:
gpg_key: "rabbitmq.gpg"
ubuntu-aarch64:
+ ubuntu:
+ distro: True
+ ubuntu-security:
+ distro: True
rabbitmq:
url: "https://deb1.rabbitmq.com/rabbitmq-server/ubuntu/noble"
suite: "noble"
@@ -106,8 +123,7 @@ ubuntu-aarch64:
rpm:
ceph:
- gpgkey: "https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-Storage"
- metalink: "https://mirrors.centos.org/metalink?repo=centos-storage-sig-ceph-squid-10-stream&arch=$basearch"
+ distro: True
name: "centos-ceph-squid"
crb:
distro: True
@@ -179,12 +195,36 @@ rpm:
https://github.com/rabbitmq/signing-keys/releases/download/3.0/rabbitmq-release-signing-key.asc
name: "rabbitmq_rabbitmq-server"
+centos:
+ appstream:
+ distro: True
+ file_group: "centos.repo"
+ name: "appstream"
+ baseos:
+ distro: True
+ file_group: "centos.repo"
+ name: "baseos"
+ crb:
+ file_group: "centos.repo"
+
centos-aarch64:
erlang:
baseurl: "https://download.copr.fedorainfracloud.org/results/@openstack-kolla/rabbitmq-erlang-27/rhel-$releasever-aarch64/"
gpgkey: "https://download.copr.fedorainfracloud.org/results/@openstack-kolla/rabbitmq-erlang-27/pubkey.gpg"
name: "copr-rabbitmq-erlang"
+rocky:
+ appstream:
+ distro: True
+ file_group: "rocky.repo"
+ name: "appstream"
+ baseos:
+ distro: True
+ file_group: "rocky.repo"
+ name: "baseos"
+ crb:
+ file_group: "rocky.repo"
+
rocky-aarch64:
erlang:
baseurl: "https://download.copr.fedorainfracloud.org/results/@openstack-kolla/rabbitmq-erlang-27/rhel-$releasever-aarch64/"
diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py
index e1dc810d1..8a577aead 100644
--- a/kolla/tests/test_methods.py
+++ b/kolla/tests/test_methods.py
@@ -10,6 +10,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import tempfile
+
+import yaml
+
from kolla.template import methods
from kolla.tests import base
@@ -30,7 +34,9 @@ class MethodsTest(base.TestCase):
}
result = methods.handle_repos(template_vars, ["grafana"], "enable")
- expectCmd = "RUN echo '[grafana]' "
+ expectCmd = "RUN grep -rlF '[grafana]' /etc/yum.repos.d/ "
+ expectCmd += "2>/dev/null | xargs -r rm -f && "
+ expectCmd += "echo '[grafana]' "
expectCmd += ">/etc/yum.repos.d/grafana.repo && "
expectCmd += "echo 'name=grafana' "
expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
@@ -42,7 +48,7 @@ class MethodsTest(base.TestCase):
expectCmd += ">>/etc/yum.repos.d/grafana.repo"
self.assertEqual(expectCmd, result)
- def test_enable_repos_centos_metalink(self):
+ def test_enable_repos_centos_ceph_distro(self):
template_vars = {
'base_arch': 'x86_64',
'base_distro': 'centos',
@@ -50,16 +56,7 @@ class MethodsTest(base.TestCase):
}
result = methods.handle_repos(template_vars, ["ceph"], "enable")
- expectCmd = "RUN echo '[centos-ceph-squid]' "
- expectCmd += ">/etc/yum.repos.d/ceph.repo && "
- expectCmd += "echo 'name=centos-ceph-squid' "
- expectCmd += ">>/etc/yum.repos.d/ceph.repo && "
- expectCmd += "echo 'enabled=1' "
- expectCmd += ">>/etc/yum.repos.d/ceph.repo && "
- expectCmd += "echo 'gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-Storage' " # noqa: E501
- expectCmd += ">>/etc/yum.repos.d/ceph.repo && "
- expectCmd += "echo 'metalink=https://mirrors.centos.org/metalink?repo=centos-storage-sig-ceph-squid-10-stream&arch=$basearch' " # noqa: E501
- expectCmd += ">>/etc/yum.repos.d/ceph.repo"
+ expectCmd = "RUN dnf config-manager --enable centos-ceph-squid || true"
self.assertEqual(expectCmd, result)
def test_enable_repos_centos_arch(self):
@@ -70,7 +67,9 @@ class MethodsTest(base.TestCase):
}
result = methods.handle_repos(template_vars, ["grafana"], "enable")
- expectCmd = "RUN echo '[grafana]' "
+ expectCmd = "RUN grep -rlF '[grafana]' /etc/yum.repos.d/ "
+ expectCmd += "2>/dev/null | xargs -r rm -f && "
+ expectCmd += "echo '[grafana]' "
expectCmd += ">/etc/yum.repos.d/grafana.repo && "
expectCmd += "echo 'name=grafana' "
expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
@@ -91,7 +90,9 @@ class MethodsTest(base.TestCase):
result = methods.handle_repos(template_vars,
["grafana", "rabbitmq"], "enable")
- expectCmd = "RUN echo '[grafana]' "
+ expectCmd = "RUN grep -rlF '[grafana]' /etc/yum.repos.d/ "
+ expectCmd += "2>/dev/null | xargs -r rm -f && "
+ expectCmd += "echo '[grafana]' "
expectCmd += ">/etc/yum.repos.d/grafana.repo && "
expectCmd += "echo 'name=grafana' "
expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
@@ -102,6 +103,8 @@ class MethodsTest(base.TestCase):
expectCmd += "echo 'baseurl=https://rpm.grafana.com' "
expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
+ expectCmd += "grep -rlF '[rabbitmq_rabbitmq-server]' "
+ expectCmd += "/etc/yum.repos.d/ 2>/dev/null | xargs -r rm -f && "
expectCmd += "echo '[rabbitmq_rabbitmq-server]' "
expectCmd += ">/etc/yum.repos.d/rabbitmq.repo && "
expectCmd += "echo 'name=rabbitmq_rabbitmq-server' "
@@ -151,6 +154,8 @@ class MethodsTest(base.TestCase):
result = methods.handle_repos(template_vars,
['crb', 'grafana'], 'enable')
expectCmd = "RUN dnf config-manager --enable crb || true && "
+ expectCmd += "grep -rlF '[grafana]' /etc/yum.repos.d/ "
+ expectCmd += "2>/dev/null | xargs -r rm -f && "
expectCmd += "echo '[grafana]' "
expectCmd += ">/etc/yum.repos.d/grafana.repo && "
expectCmd += "echo 'name=grafana' "
@@ -168,6 +173,7 @@ class MethodsTest(base.TestCase):
"base_arch": "x86_64",
"base_distro": "debian",
"base_package_type": "deb",
+ "openstack_release_codename": "Gazpacho",
}
result = methods.handle_repos(template_vars, ["grafana"], "enable")
@@ -188,6 +194,7 @@ class MethodsTest(base.TestCase):
'base_arch': 'aarch64',
'base_distro': 'debian',
'base_package_type': 'deb',
+ 'openstack_release_codename': 'Gazpacho',
}
result = methods.handle_repos(template_vars, ["rabbitmq"], "enable")
@@ -220,6 +227,7 @@ class MethodsTest(base.TestCase):
'base_arch': 'x86_64',
'base_distro': 'debian',
'base_package_type': 'deb',
+ 'openstack_release_codename': 'Gazpacho',
}
result = methods.handle_repos(template_vars,
@@ -273,3 +281,179 @@ class MethodsTest(base.TestCase):
r'repositories',
methods.handle_repos, template_vars, 'grafana',
'disable')
+
+ def test_enable_repos_debian_distro_noop(self):
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'debian',
+ 'base_package_type': 'deb',
+ }
+
+ result = methods.handle_repos(template_vars, ['debian'], 'enable')
+ self.assertEqual('', result)
+
+ def test_enable_repos_deb_absolute_signed_by(self):
+ repos = {'debian': {'test-repo': {
+ 'url': 'http://example.com/debian',
+ 'suite': 'trixie',
+ 'component': 'main',
+ 'gpg_key': '/usr/share/keyrings/test.gpg',
+ }}}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'debian',
+ 'base_package_type': 'deb',
+ 'openstack_release_codename': 'Gazpacho',
+ 'repos_yaml': f.name,
+ }
+ result = methods.handle_repos(
+ template_vars, ['test-repo'], 'enable')
+ self.assertIn(
+ 'Signed-By: /usr/share/keyrings/test.gpg', result)
+
+ def test_repos_yaml_merge_keeps_default_repos(self):
+ repos = {'rocky': {'baseos': {
+ 'name': 'baseos',
+ 'baseurl': 'http://mirror.example.com/rocky/10/BaseOS/$basearch/',
+ 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10',
+ }}}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'centos',
+ 'base_package_type': 'rpm',
+ 'repos_yaml': f.name,
+ }
+ expected_commands = methods.handle_repos(
+ template_vars, ['grafana'], 'enable')
+ self.assertIn(
+ "echo 'gpgkey=https://rpm.grafana.com/gpg.key' "
+ ">>/etc/yum.repos.d/grafana.repo",
+ expected_commands)
+
+ def test_repos_yaml_distro_override_validation(self):
+ """Overriding baseos and appstream would delete rocky.repo.
+
+ crb is a distro repo that also uses that file group. Verify a
+ ValueError is raised so the user knows to override crb as well.
+ """
+ repos = {'rocky': {
+ 'baseos': {
+ 'name': 'baseos',
+ 'baseurl': 'http://mirror.example.com/rocky/10/BaseOS/',
+ 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10',
+ },
+ 'appstream': {
+ 'name': 'appstream',
+ 'baseurl': 'http://mirror.example.com/rocky/10/AppStream/',
+ 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10',
+ },
+ }}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'rocky',
+ 'base_package_type': 'rpm',
+ 'repos_yaml': f.name,
+ }
+ self.assertRaisesRegex(
+ ValueError, "still using distro defaults and share the same",
+ methods.handle_repos, template_vars, ['baseos'], 'enable')
+
+ def test_enable_repos_rpm_missing_url_raises(self):
+ repos = {'rpm': {'test-repo': {'name': 'test-repo'}}}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'centos',
+ 'base_package_type': 'rpm',
+ 'repos_yaml': f.name,
+ }
+ self.assertRaises(ValueError, methods.handle_repos,
+ template_vars, ['test-repo'], 'enable')
+
+ def test_enable_repos_rpm_missing_gpgkey_raises(self):
+ repos = {'rpm': {'test-repo': {
+ 'name': 'test-repo',
+ 'baseurl': 'http://mirror.example.com/repo/',
+ }}}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'centos',
+ 'base_package_type': 'rpm',
+ 'repos_yaml': f.name,
+ }
+ self.assertRaises(ValueError, methods.handle_repos,
+ template_vars, ['test-repo'], 'enable')
+
+ def test_repos_yaml_rpm_section_override_not_blocked_by_distro_section(
+ self):
+ """Overriding distro repos in 'rpm' section must not be undone.
+
+ The more-specific distro section must not merge distro:True back
+ on top of a URL-bearing override from the generic 'rpm' section.
+ """
+ base = 'http://mirror.example.com/centos/10'
+ gpgkey = 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial'
+ repos = {'rpm': {
+ 'baseos': {
+ 'name': 'baseos',
+ 'baseurl': base + '/BaseOS/$basearch/',
+ 'gpgkey': gpgkey,
+ },
+ 'appstream': {
+ 'name': 'appstream',
+ 'baseurl': base + '/AppStream/$basearch/',
+ 'gpgkey': gpgkey,
+ },
+ 'crb': {
+ 'name': 'crb',
+ 'baseurl': base + '/CRB/$basearch/',
+ 'gpgkey': gpgkey,
+ },
+ }}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'centos',
+ 'base_package_type': 'rpm',
+ 'repos_yaml': f.name,
+ }
+ result = methods.handle_repos(
+ template_vars, ['baseos', 'appstream', 'crb'], 'enable')
+ self.assertIn('mirror.example.com', result)
+
+ def test_enable_repos_openstack_release_codename_substitution(self):
+ repos = {'ubuntu': {'ubuntu-cloud-archive': {
+ 'url': 'http://ubuntu-cloud.archive.canonical.com/ubuntu',
+ 'suite': 'noble-updates/{openstack_release_codename}',
+ 'component': 'main',
+ 'gpg_key': '/usr/share/keyrings/ubuntu-cloud-keyring.gpg',
+ }}}
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+ yaml.dump(repos, f)
+ f.flush()
+ template_vars = {
+ 'base_arch': 'x86_64',
+ 'base_distro': 'ubuntu',
+ 'base_package_type': 'deb',
+ 'openstack_release_codename': 'Gazpacho',
+ 'repos_yaml': f.name,
+ }
+ result = methods.handle_repos(
+ template_vars, ['ubuntu-cloud-archive'], 'enable')
+ self.assertIn('noble-updates/gazpacho', result)
diff --git a/releasenotes/notes/base-repos-in-yaml-f6a4a871da92c823.yaml b/releasenotes/notes/base-repos-in-yaml-f6a4a871da92c823.yaml
new file mode 100644
index 000000000..1f98f7d60
--- /dev/null
+++ b/releasenotes/notes/base-repos-in-yaml-f6a4a871da92c823.yaml
@@ -0,0 +1,23 @@
+---
+features:
+ - |
+ Base OS repositories for CentOS Stream and Rocky Linux (``baseos``,
+ ``appstream``, ``crb``) are now defined in ``repos.yaml`` and can be
+ overridden via ``--repos-yaml`` to point at local mirrors. When
+ overriding any of these repositories, all others sharing the same
+ ``.repo`` file (identified by the ``file_group`` field) must also be
+ overridden to avoid silently disabling them.
+ - |
+ Base OS repositories for Debian and Ubuntu are now defined in
+ ``repos.yaml`` with ``distro: True``. They can be overridden via
+ ``--repos-yaml`` to point at local mirrors. The ``gpg_key`` field for
+ Debian/Ubuntu repositories now supports absolute paths, allowing use of
+ distro-provided keyrings such as
+ ``/usr/share/keyrings/debian-archive-keyring.gpg``.
+ - |
+ When overriding an RPM base repository, any existing ``.repo`` file on
+ the image containing that repository ID is automatically removed before
+ the new one is created.
+ - |
+ User-provided ``repos.yaml`` is now merged with upstream default instead of
+ requiring a full copy of stock file.
diff --git a/roles/kolla-build/defaults/main.yml b/roles/kolla-build/defaults/main.yml
index fe7c33a33..98051c97b 100644
--- a/roles/kolla-build/defaults/main.yml
+++ b/roles/kolla-build/defaults/main.yml
@@ -1,4 +1,5 @@
---
kolla_build_logs_dir: "{{ zuul_output_dir }}/logs/kolla/build"
+kolla_build_repos_yaml_path: ""
kolla_build_template_overrides_path: ""
kolla_build_venv_path: "/tmp/kolla-virtualenv"
diff --git a/roles/kolla-build/tasks/main.yml b/roles/kolla-build/tasks/main.yml
index e5529f381..927bb1309 100644
--- a/roles/kolla-build/tasks/main.yml
+++ b/roles/kolla-build/tasks/main.yml
@@ -1,4 +1,13 @@
---
+- name: Template repos.yaml
+ ansible.builtin.template:
+ src: "{{ kolla_build_repos_yaml_path }}"
+ dest: /etc/kolla/repos.yaml
+ mode: "0660"
+ owner: zuul
+ group: zuul
+ when: kolla_build_repos_yaml_path | length > 0
+
- name: Template template_overrides.j2
ansible.builtin.template:
src: "{{ kolla_build_template_overrides_path }}"
diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml
index ad3c80075..99791da0f 100644
--- a/tests/playbooks/run.yml
+++ b/tests/playbooks/run.yml
@@ -62,6 +62,16 @@
when:
- not publisher
+ - name: Set up base repo overrides
+ vars:
+ kolla_repos_yaml_config:
+ DEFAULT:
+ repos_yaml: /etc/kolla/repos.yaml
+ ansible.builtin.set_fact:
+ kolla_build_config: "{{ kolla_build_config | combine(kolla_repos_yaml_config, recursive=True) }}"
+ kolla_build_repos_yaml_path: "{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/openstack/kolla'].src_dir }}/tests/templates/repos.yaml.j2"
+ when: base_distro in ['debian', 'rocky', 'ubuntu']
+
- name: Import kolla-build-config role
import_role:
name: kolla-build-config
diff --git a/tests/templates/repos.yaml.j2 b/tests/templates/repos.yaml.j2
new file mode 100644
index 000000000..42c6724f9
--- /dev/null
+++ b/tests/templates/repos.yaml.j2
@@ -0,0 +1,63 @@
+---
+{% if base_distro == 'debian' %}
+debian:
+ debian:
+ component: "main"
+ gpg_key: "/usr/share/keyrings/debian-archive-keyring.gpg"
+ suite: "trixie trixie-updates trixie-backports"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/debian"
+ debian-security:
+ component: "main"
+ gpg_key: "/usr/share/keyrings/debian-archive-keyring.gpg"
+ suite: "trixie-security"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/debian-security"
+{% elif base_distro == 'rocky' %}
+rocky:
+ appstream:
+ baseurl: "http://dl.rockylinux.org/$contentdir/$releasever/AppStream/$basearch/os/"
+ gpgkey: "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10"
+ name: "appstream"
+ baseos:
+ baseurl: "http://dl.rockylinux.org/$contentdir/$releasever/BaseOS/$basearch/os/"
+ gpgkey: "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10"
+ name: "baseos"
+ crb:
+ baseurl: "http://dl.rockylinux.org/$contentdir/$releasever/CRB/$basearch/os/"
+ gpgkey: "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10"
+ name: "crb"
+{% elif base_distro == 'ubuntu' %}
+ubuntu:
+ ubuntu:
+ component: "main universe"
+ gpg_key: "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+ suite: "noble noble-updates noble-backports"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/ubuntu/"
+ ubuntu-security:
+ component: "main universe"
+ gpg_key: "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+ suite: "noble-security"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/ubuntu/"
+ ubuntu-cloud-archive:
+ component: "main"
+ gpg_key: "/usr/share/keyrings/ubuntu-cloud-keyring.gpg"
+ suite: "noble-updates/gazpacho"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive"
+ubuntu-aarch64:
+ ubuntu:
+ component: "main universe"
+ gpg_key: "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+ suite: "noble noble-updates noble-backports"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/ubuntu-ports/"
+ ubuntu-security:
+ component: "main universe"
+ gpg_key: "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+ suite: "noble-security"
+ trusted: True
+ url: "http://{{ nodepool_mirror_host }}/ubuntu-ports/"
+{% endif %}
diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2
index ee67e2a62..3baa8d704 100644
--- a/tests/templates/template_overrides.j2
+++ b/tests/templates/template_overrides.j2
@@ -14,42 +14,9 @@ ENV PIP_EXTRA_INDEX_URL {{ nodepool_wheel_mirror }}
{% endblock %}
{% endraw %}
-{% if base_distro == 'rocky' %}
-{% raw %}
-{% block base_centos_repo_overrides_post_copy %}
-{% endraw %}
-RUN sed -i -e "s/^mirrorlist/#mirrorlist/g" -e "s/^#baseurl/baseurl/g" \
- /etc/yum.repos.d/rocky.repo
-{% raw %}
-{% endblock %}
-{% endraw %}
-{% endif %}
-
-{% raw %}
-{% block base_debian_after_sources_list %}
-{% endraw %}
-{% if base_distro == "debian" %}
-
-RUN cp /etc/apt/sources.list.d/debian.sources /tmp/ && \
- sed -i -e "s|^URIs: http://deb.debian.org/debian$|URIs: http://{{ nodepool_mirror_host }}/debian\nTrusted: yes|" \
- -e "s|^URIs: http://deb.debian.org/debian-security|URIs: http://{{ nodepool_mirror_host }}/debian-security\nTrusted: yes|" \
- /etc/apt/sources.list.d/debian.sources
-
-{% elif base_distro == "ubuntu" %}
-
-RUN cp /etc/apt/sources.list.d/ubuntu.sources /tmp/ && \
- sed -i -e "s|^URIs: http://archive.ubuntu.com/ubuntu/$|URIs: http://{{ nodepool_mirror_host }}/ubuntu/\nTrusted: yes|" \
- -e "s|^URIs: http://security.ubuntu.com/ubuntu/$|URIs: http://{{ nodepool_mirror_host }}/ubuntu/\nTrusted: yes|" \
- -e "s|^URIs: http://ports.ubuntu.com/ubuntu-ports/$|URIs: http://{{ nodepool_mirror_host }}/ubuntu-ports/\nTrusted: yes|" \
- -e "s|^URIs: http://ubuntu-cloud.archive.canonical.com/ubuntu$|URIs: http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive\nTrusted: yes|" \
- /etc/apt/sources.list.d/ubuntu.sources
-
-{% endif %}
-{% raw %}
-{% endblock %}
-
{# Revert to upstream mirrors after build is complete #}
+{% raw %}
{% block footer %}
{% endraw %}
@@ -59,15 +26,6 @@ ENV PIP_TRUSTED_HOST=
ENV PIP_EXTRA_INDEX_URL=
{% endif %}
-{% if base_distro == "debian" %}
-RUN if [ -f /tmp/debian.sources ]; then \
- mv -f /tmp/debian.sources /etc/apt/sources.list.d/debian.sources; \
- fi
-{% elif base_distro == "ubuntu" %}
-RUN if [ -f /tmp/ubuntu.sources ]; then \
- mv -f /tmp/ubuntu.sources /etc/apt/sources.list.d/ubuntu.sources; \
- fi
-{% endif %}
{% raw %}
{% endblock %}
{% endraw %}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-05 16:51:12 +0000