From 3f01507eb215b2162e9815257cbd2ba4091251df Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 17 Aug 2026 08:02:45 +0200 Subject: CI: Switch to build from Zuul cached repos Instead of overloading tarballs.opendev.org (and relying on their functional state), let's build images directly from Zuul cached workspaces. Adding a script that will validate if we're missing anything when adding new images. Change-Id: Ie1d8dce841b9a996e96e3a6e1b097b0d256c0bea Signed-off-by: Michal Nasiadka --- tools/validate-build-sources-overrides.py | 175 ++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 tools/validate-build-sources-overrides.py (limited to 'tools') diff --git a/tools/validate-build-sources-overrides.py b/tools/validate-build-sources-overrides.py new file mode 100644 index 000000000..212c380c2 --- /dev/null +++ b/tools/validate-build-sources-overrides.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check that every buildable-from-source entry in kolla/common/sources.py +has a matching CI override in +roles/kolla-build-config/defaults/main.yml (kolla_build_sources), that +each such project is declared in the kolla-base job's required-projects +in zuul.d/base.yaml, and that any pinned git reference in sources.py +matches the override-checkout used to fetch it in CI. This ensures CI +builds every source from a Zuul-cached git checkout instead of +downloading a tarball from tarballs.opendev.org. +""" + +import os +import sys + +import yaml + +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(PROJECT_ROOT) + +from kolla.common.sources import SOURCES # noqa: E402 + +BUILD_SOURCES_PATH = os.path.join( + PROJECT_ROOT, 'roles', 'kolla-build-config', 'defaults', 'main.yml') +ZUUL_BASE_PATH = os.path.join(PROJECT_ROOT, 'zuul.d', 'base.yaml') + +# Sources deliberately not covered by kolla_build_sources, with the +# reason a git-checkout override doesn't apply to them. +EXCLUDED_SOURCES = { + # Binary releases downloaded from GitHub, not built from a source + # checkout. + 'etcd', + 'letsencrypt-lego', + 'magnum-conductor-plugin-helm', + # GitHub projects that are not registered with the OpenStack Zuul + # tenant (unlike gnocchixyz/gnocchi), so they can't be required-projects. + 'kerbside-base', + 'mariadb-server-plugin-mariadb-docker', +} + + +def is_excluded(name): + return name in EXCLUDED_SOURCES or name.startswith('prometheus-') + + +def as_list(section_names): + if isinstance(section_names, str): + return [section_names] + return list(section_names) + + +def load_kolla_build_sources(): + with open(BUILD_SOURCES_PATH) as f: + return yaml.safe_load(f)['kolla_build_sources'] + + +def load_required_projects(): + """Parse the kolla-base job's required-projects in zuul.d/base.yaml. + + Returns (all project names, {project: override-checkout}). + """ + with open(ZUUL_BASE_PATH) as f: + docs = yaml.safe_load(f) + + job = next(doc['job'] for doc in docs + if 'job' in doc and doc['job']['name'] == 'kolla-base') + + all_projects = set() + checkouts = {} + for entry in job.get('required-projects', []): + if isinstance(entry, str): + all_projects.add(entry) + else: + all_projects.add(entry['name']) + if 'override-checkout' in entry: + checkouts[entry['name']] = entry['override-checkout'] + return all_projects, checkouts + + +def main(): + kolla_build_sources = load_kolla_build_sources() + required_projects, override_checkouts = load_required_projects() + + covered = set() + for section_names in kolla_build_sources.values(): + covered.update(as_list(section_names)) + + buildable = {name for name in SOURCES if not is_excluded(name)} + + missing = sorted(buildable - covered) + stale = sorted(covered - set(SOURCES)) + redundant_exclusions = sorted( + name for name in EXCLUDED_SOURCES if name not in SOURCES) + not_required = sorted(set(kolla_build_sources) - required_projects) + + ref_errors = [] + for project, section_names in sorted(kolla_build_sources.items()): + for section in as_list(section_names): + info = SOURCES.get(section) + if not info or info.get('type') != 'git': + continue + expected_ref = info.get('reference') + actual = override_checkouts.get(project) + if actual is None: + ref_errors.append( + f"{project} ({section}) is pinned to reference " + f"'{expected_ref}' in sources.py but has no " + "override-checkout in zuul.d/base.yaml " + "required-projects") + elif actual != expected_ref: + ref_errors.append( + f"{project} ({section}): sources.py reference " + f"'{expected_ref}' != required-projects " + f"override-checkout '{actual}'") + + for project, checkout in sorted(override_checkouts.items()): + section_names = kolla_build_sources.get(project) + if section_names is None: + continue + git_sections = [s for s in as_list(section_names) + if SOURCES.get(s, {}).get('type') == 'git'] + if not git_sections: + ref_errors.append( + f"{project} has override-checkout '{checkout}' in " + "required-projects but none of its kolla_build_sources " + "sections are type 'git' in sources.py") + + if missing: + print("ERROR: sources.py entries missing a kolla_build_sources " + "override in roles/kolla-build-config/defaults/main.yml:") + for name in missing: + print(f" {name}") + + if stale: + print("ERROR: kolla_build_sources overrides referencing sections " + "that no longer exist in kolla/common/sources.py:") + for name in stale: + print(f" {name}") + + if redundant_exclusions: + print("ERROR: EXCLUDED_SOURCES entries referencing sections that " + "no longer exist in kolla/common/sources.py, remove them:") + for name in redundant_exclusions: + print(f" {name}") + + if not_required: + print("ERROR: kolla_build_sources projects missing from the " + "kolla-base job's required-projects in zuul.d/base.yaml:") + for name in not_required: + print(f" {name}") + + if ref_errors: + print("ERROR: git reference mismatches between sources.py and " + "zuul.d/base.yaml required-projects:") + for msg in ref_errors: + print(f" {msg}") + + return 1 if (missing or stale or redundant_exclusions or not_required + or ref_errors) else 0 + + +if __name__ == "__main__": + sys.exit(main()) -- cgit v1.2.3