CI: Switch to build from Zuul cached repos

Instead of overloading tarballs.opendev.org (and relying on their
functional state), let's build images directly from Zuul cached
workspaces.

Adding a script that will validate if we're missing anything when
adding new images.

Change-Id: Ie1d8dce841b9a996e96e3a6e1b097b0d256c0bea
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>

5 files changed, 275 insertions, 29 deletions

diff --git a/roles/kolla-build-config/defaults/main.yml b/roles/kolla-build-config/defaults/main.yml
index 38eae184c..ced1ac659 100644
--- a/roles/kolla-build-config/defaults/main.yml
+++ b/roles/kolla-build-config/defaults/main.yml
@@ -11,58 +11,70 @@ kolla_build_config: {}
 
 # Dict mapping project names to sections in kolla-build.conf.
 kolla_build_sources:
-  openstack/requirements: openstack-base
-
+  gnocchixyz/gnocchi: gnocchi-base
   openstack/aodh: aodh-base
   openstack/barbican: barbican-base
   openstack/bifrost: bifrost-base
   openstack/blazar: blazar-base
+  openstack/blazar-dashboard: horizon-plugin-blazar-dashboard
   openstack/blazar-nova: nova-base-plugin-blazar
   openstack/ceilometer: ceilometer-base
   openstack/cinder: cinder-base
   openstack/cloudkitty: cloudkitty-base
+  openstack/cloudkitty-dashboard: horizon-plugin-cloudkitty-dashboard
+  openstack/cyborg: cyborg-base
   openstack/designate: designate-base
+  openstack/designate-dashboard: horizon-plugin-designate-dashboard
   openstack/glance: glance-base
-  openstack/gnocchi: gnocchi-base
   openstack/heat: heat-base
+  openstack/heat-dashboard: horizon-plugin-heat-dashboard
+  openstack/horizon: horizon
   openstack/ironic: ironic-base
+  openstack/ironic-prometheus-exporter: ironic-conductor-plugin-prometheus-exporter
+  openstack/ironic-ui: horizon-plugin-ironic-ui
   openstack/keystone: keystone-base
   openstack/magnum: magnum-base
+  openstack/magnum-ui: horizon-plugin-magnum-ui
   openstack/manila: manila-base
+  openstack/manila-ui: horizon-plugin-manila-ui
+  openstack/masakari: masakari-base
+  openstack/masakari-dashboard: horizon-plugin-masakari-dashboard
+  openstack/masakari-monitors: masakari-monitors
   openstack/mistral: mistral-base
-  openstack/neutron: neutron-base
+  openstack/mistral-dashboard: horizon-plugin-mistral-dashboard
   openstack/networking-baremetal: neutron-base-plugin-networking-baremetal
-  openstack/neutron-dynamic-routing:
-    - neutron-bgp-dragent
-    - neutron-server-plugin-neutron-dynamic-routing
   openstack/networking-generic-switch: neutron-base-plugin-networking-generic-switch
   openstack/networking-sfc:
     - neutron-base-plugin-networking-sfc
     - tacker-base-plugin-networking-sfc
+  openstack/neutron: neutron-base
+  openstack/neutron-dynamic-routing:
+    - neutron-bgp-dragent
+    - neutron-server-plugin-neutron-dynamic-routing
+  openstack/neutron-fwaas: neutron-base-plugin-neutron-fwaas
+  openstack/neutron-fwaas-dashboard: horizon-plugin-fwaas-dashboard
   openstack/neutron-vpnaas:
     - neutron-vpnaas-agent
     - neutron-base-plugin-vpnaas-agent
+  openstack/neutron-vpnaas-dashboard: horizon-plugin-neutron-vpnaas-dashboard
   openstack/nova: nova-base
   openstack/octavia: octavia-base
-  openstack/skyline: skyline-base
+  openstack/octavia-dashboard: horizon-plugin-octavia-dashboard
+  openstack/ovn-octavia-provider:
+    - octavia-api-plugin-ovn-octavia-provider
+    - octavia-driver-agent-plugin-ovn-octavia-provider
+  openstack/placement: placement-base
+  openstack/requirements:
+    - openstack-base
+    - kolla-toolbox
+  openstack/skyline-apiserver: skyline-apiserver
+  openstack/skyline-console: skyline-console
   openstack/tacker:
     - tacker-base
     - mistral-base-plugin-tacker
-  openstack/trove: trove-base
-  openstack/watcher: watcher-base
-
-  # Horizon and its plugins
-  openstack/horizon: horizon
-  openstack/blazar-dashboard: horizon-plugin-blazar-dashboard
-  openstack/cloudkitty-dashboard: horizon-plugin-cloudkitty-dashboard
-  openstack/designate-dashboard: horizon-plugin-designate-dashboard
-  openstack/heat-dashboard: horizon-plugin-heat-dashboard
-  openstack/ironic-ui: horizon-plugin-ironic-ui
-  openstack/magnum-ui: horizon-plugin-magnum-ui
-  openstack/manila-ui: horizon-plugin-manila-ui
-  openstack/mistral-dashboard: horizon-plugin-mistral-dashboard
-  openstack/neutron-vpnaas-dashboard: horizon-plugin-neutron-vpnaas-dashboard
-  openstack/octavia-dashboard: horizon-plugin-octavia-dashboard
   openstack/tacker-horizon: horizon-plugin-tacker-dashboard
+  openstack/tap-as-a-service: neutron-base-plugin-tap-as-a-service
+  openstack/trove: trove-base
   openstack/trove-dashboard: horizon-plugin-trove-dashboard
+  openstack/watcher: watcher-base
   openstack/watcher-dashboard: horizon-plugin-watcher-dashboard
diff --git a/roles/kolla-build-config/templates/kolla-build.conf.j2 b/roles/kolla-build-config/templates/kolla-build.conf.j2
index bfc5dd426..07a50baae 100644
--- a/roles/kolla-build-config/templates/kolla-build.conf.j2
+++ b/roles/kolla-build-config/templates/kolla-build.conf.j2
@@ -7,20 +7,21 @@
 {% endfor %}
 {% endfor %}
 
-{# Generate a section for every build source for which there is a Zuul change. #}
+{# Generate a section for every build source Zuul has a checkout for,
+   whether it's a required-project or a Depends-On. #}
 
-{% for zuul_item in zuul['items'] %}
-{% if zuul_item.project.name in kolla_build_sources %}
-{% set source_names = kolla_build_sources[zuul_item.project.name] %}
+{% for canonical_name, project in zuul['projects'].items() %}
+{% if project.name in kolla_build_sources %}
+{% set source_names = kolla_build_sources[project.name] %}
 {% if source_names is string %}
 [{{ source_names }}]
 type = local
-location = {{ ansible_env.PWD ~ '/' ~ zuul_item.project.src_dir }}
+location = {{ ansible_env.PWD ~ '/' ~ project.src_dir }}
 {% else %}
 {% for source_name in source_names %}
 [{{ source_name }}]
 type = local
-location = {{ ansible_env.PWD ~ '/' ~ zuul_item.project.src_dir }}
+location = {{ ansible_env.PWD ~ '/' ~ project.src_dir }}
 {% endfor %}
 {% endif %}
 {% endif %}
diff --git a/tools/validate-build-sources-overrides.py b/tools/validate-build-sources-overrides.py
new file mode 100644
index 000000000..212c380c2
--- /dev/null
+++ b/tools/validate-build-sources-overrides.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Check that every buildable-from-source entry in kolla/common/sources.py
+has a matching CI override in
+roles/kolla-build-config/defaults/main.yml (kolla_build_sources), that
+each such project is declared in the kolla-base job's required-projects
+in zuul.d/base.yaml, and that any pinned git reference in sources.py
+matches the override-checkout used to fetch it in CI. This ensures CI
+builds every source from a Zuul-cached git checkout instead of
+downloading a tarball from tarballs.opendev.org.
+"""
+
+import os
+import sys
+
+import yaml
+
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(PROJECT_ROOT)
+
+from kolla.common.sources import SOURCES  # noqa: E402
+
+BUILD_SOURCES_PATH = os.path.join(
+    PROJECT_ROOT, 'roles', 'kolla-build-config', 'defaults', 'main.yml')
+ZUUL_BASE_PATH = os.path.join(PROJECT_ROOT, 'zuul.d', 'base.yaml')
+
+# Sources deliberately not covered by kolla_build_sources, with the
+# reason a git-checkout override doesn't apply to them.
+EXCLUDED_SOURCES = {
+    # Binary releases downloaded from GitHub, not built from a source
+    # checkout.
+    'etcd',
+    'letsencrypt-lego',
+    'magnum-conductor-plugin-helm',
+    # GitHub projects that are not registered with the OpenStack Zuul
+    # tenant (unlike gnocchixyz/gnocchi), so they can't be required-projects.
+    'kerbside-base',
+    'mariadb-server-plugin-mariadb-docker',
+}
+
+
+def is_excluded(name):
+    return name in EXCLUDED_SOURCES or name.startswith('prometheus-')
+
+
+def as_list(section_names):
+    if isinstance(section_names, str):
+        return [section_names]
+    return list(section_names)
+
+
+def load_kolla_build_sources():
+    with open(BUILD_SOURCES_PATH) as f:
+        return yaml.safe_load(f)['kolla_build_sources']
+
+
+def load_required_projects():
+    """Parse the kolla-base job's required-projects in zuul.d/base.yaml.
+
+    Returns (all project names, {project: override-checkout}).
+    """
+    with open(ZUUL_BASE_PATH) as f:
+        docs = yaml.safe_load(f)
+
+    job = next(doc['job'] for doc in docs
+               if 'job' in doc and doc['job']['name'] == 'kolla-base')
+
+    all_projects = set()
+    checkouts = {}
+    for entry in job.get('required-projects', []):
+        if isinstance(entry, str):
+            all_projects.add(entry)
+        else:
+            all_projects.add(entry['name'])
+            if 'override-checkout' in entry:
+                checkouts[entry['name']] = entry['override-checkout']
+    return all_projects, checkouts
+
+
+def main():
+    kolla_build_sources = load_kolla_build_sources()
+    required_projects, override_checkouts = load_required_projects()
+
+    covered = set()
+    for section_names in kolla_build_sources.values():
+        covered.update(as_list(section_names))
+
+    buildable = {name for name in SOURCES if not is_excluded(name)}
+
+    missing = sorted(buildable - covered)
+    stale = sorted(covered - set(SOURCES))
+    redundant_exclusions = sorted(
+        name for name in EXCLUDED_SOURCES if name not in SOURCES)
+    not_required = sorted(set(kolla_build_sources) - required_projects)
+
+    ref_errors = []
+    for project, section_names in sorted(kolla_build_sources.items()):
+        for section in as_list(section_names):
+            info = SOURCES.get(section)
+            if not info or info.get('type') != 'git':
+                continue
+            expected_ref = info.get('reference')
+            actual = override_checkouts.get(project)
+            if actual is None:
+                ref_errors.append(
+                    f"{project} ({section}) is pinned to reference "
+                    f"'{expected_ref}' in sources.py but has no "
+                    "override-checkout in zuul.d/base.yaml "
+                    "required-projects")
+            elif actual != expected_ref:
+                ref_errors.append(
+                    f"{project} ({section}): sources.py reference "
+                    f"'{expected_ref}' != required-projects "
+                    f"override-checkout '{actual}'")
+
+    for project, checkout in sorted(override_checkouts.items()):
+        section_names = kolla_build_sources.get(project)
+        if section_names is None:
+            continue
+        git_sections = [s for s in as_list(section_names)
+                        if SOURCES.get(s, {}).get('type') == 'git']
+        if not git_sections:
+            ref_errors.append(
+                f"{project} has override-checkout '{checkout}' in "
+                "required-projects but none of its kolla_build_sources "
+                "sections are type 'git' in sources.py")
+
+    if missing:
+        print("ERROR: sources.py entries missing a kolla_build_sources "
+              "override in roles/kolla-build-config/defaults/main.yml:")
+        for name in missing:
+            print(f"  {name}")
+
+    if stale:
+        print("ERROR: kolla_build_sources overrides referencing sections "
+              "that no longer exist in kolla/common/sources.py:")
+        for name in stale:
+            print(f"  {name}")
+
+    if redundant_exclusions:
+        print("ERROR: EXCLUDED_SOURCES entries referencing sections that "
+              "no longer exist in kolla/common/sources.py, remove them:")
+        for name in redundant_exclusions:
+            print(f"  {name}")
+
+    if not_required:
+        print("ERROR: kolla_build_sources projects missing from the "
+              "kolla-base job's required-projects in zuul.d/base.yaml:")
+        for name in not_required:
+            print(f"  {name}")
+
+    if ref_errors:
+        print("ERROR: git reference mismatches between sources.py and "
+              "zuul.d/base.yaml required-projects:")
+        for msg in ref_errors:
+            print(f"  {msg}")
+
+    return 1 if (missing or stale or redundant_exclusions or not_required
+                 or ref_errors) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tox.ini b/tox.ini
index 774822b94..e8f67d8a1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,6 +52,7 @@ commands =
   flake8 {posargs}
   bash {toxinidir}/tools/validate-all-dockerfiles.sh
   python {toxinidir}/tools/validate-all-file.py
+  python {toxinidir}/tools/validate-build-sources-overrides.py
   bandit -r docker kolla tests tools
   codespell -I {toxinidir}/.codespell-ignore -x zuul.d/secrets.yaml
   ansible-galaxy collection install -r test-requirements.yml
diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml
index 8d58c81f0..21cc98371 100644
--- a/zuul.d/base.yaml
+++ b/zuul.d/base.yaml
@@ -10,6 +10,63 @@
     attempts: 5
     roles:
       - zuul: openstack/ansible-collection-kolla
+    required-projects:
+      - name: gnocchixyz/gnocchi
+        override-checkout: 4.7.0
+      - openstack/aodh
+      - openstack/barbican
+      - openstack/bifrost
+      - openstack/blazar
+      - openstack/blazar-dashboard
+      - openstack/blazar-nova
+      - openstack/ceilometer
+      - openstack/cinder
+      - openstack/cloudkitty
+      - openstack/cloudkitty-dashboard
+      - openstack/cyborg
+      - openstack/designate
+      - openstack/designate-dashboard
+      - openstack/glance
+      - openstack/heat
+      - openstack/heat-dashboard
+      - openstack/horizon
+      - openstack/ironic
+      - openstack/ironic-prometheus-exporter
+      - openstack/ironic-ui
+      - openstack/keystone
+      - openstack/magnum
+      - openstack/magnum-ui
+      - openstack/manila
+      - openstack/manila-ui
+      - openstack/masakari
+      - openstack/masakari-dashboard
+      - openstack/masakari-monitors
+      - openstack/mistral
+      - openstack/mistral-dashboard
+      - openstack/networking-baremetal
+      - openstack/networking-generic-switch
+      - openstack/networking-sfc
+      - openstack/neutron
+      - openstack/neutron-dynamic-routing
+      - openstack/neutron-fwaas
+      - openstack/neutron-fwaas-dashboard
+      - openstack/neutron-vpnaas
+      - openstack/neutron-vpnaas-dashboard
+      - openstack/nova
+      - openstack/octavia
+      - openstack/octavia-dashboard
+      - openstack/ovn-octavia-provider
+      - openstack/placement
+      - openstack/requirements
+      - openstack/skyline-apiserver
+      - openstack/skyline-console
+      - openstack/tacker
+      - openstack/tacker-horizon
+      - openstack/tap-as-a-service
+      - openstack/trove
+      - openstack/trove-dashboard
+      - openstack/watcher
+      - openstack/watcher-dashboard
     irrelevant-files:
       - ^.*\.md$
       - ^.*\.rst$