Switch sources.py into sources.yaml

Needed-By: https://review.opendev.org/c/openstack/project-config/+/998419

Change-Id: Ib00fc1a1cb2aebddf8f6a7badade2946aef31d25
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>

2 files changed, 64 insertions, 39 deletions

diff --git a/tools/check_sources.py b/tools/check_sources.py
index 189ddb170..38fc03acc 100644
--- a/tools/check_sources.py
+++ b/tools/check_sources.py
@@ -19,10 +19,7 @@ import sys
 import textwrap
 
 import requests
-
-sys.path.append(os.getcwd())
-
-from kolla.common.sources import SOURCES  # noqa: E402
+import yaml
 
 
 def github_headers():
@@ -202,14 +199,9 @@ def get_helm_sh_hashes(version, arch_map):
     return hashes
 
 
-SOURCES_PATH = os.path.normpath(
-    os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                 '..', 'kolla', 'common', 'sources.py'))
-
-
-def apply_update(name, info, new_v, new_hashes):
-    """Update version and sha256 entries for one source in sources.py."""
-    with open(SOURCES_PATH) as f:
+def apply_update(name, info, new_v, new_hashes, sources_path):
+    """Update version and sha256 entries for one source in sources.yaml."""
+    with open(sources_path) as f:
         content = f.read()
 
     version_key = 'reference' if info.get('type') == 'git' else 'version'
@@ -217,26 +209,23 @@ def apply_update(name, info, new_v, new_hashes):
     if old_v:
         new_v_str = ('v' + new_v) if old_v.startswith('v') else new_v
         content = content.replace(
-            f"'{version_key}': '{old_v}'",
-            f"'{version_key}': '{new_v_str}'", 1)
+            f'{version_key}: "{old_v}"',
+            f'{version_key}: "{new_v_str}"', 1)
 
     for arch, new_sha in new_hashes.items():
         old_sha = info.get('sha256', {}).get(arch, '')
         if old_sha and old_sha != new_sha:
             content = content.replace(old_sha, new_sha)
 
-    with open(SOURCES_PATH, 'w') as f:
+    with open(sources_path, 'w') as f:
         f.write(content)
 
 
-REPO_ROOT = os.path.normpath(
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
-
-
-def create_releasenote(changes):
+def create_releasenote(changes, repo_root):
     """Create a reno release note listing all updated components.
 
     :param changes: list of (name, old_version, new_version) tuples
+    :param repo_root: root of the kolla repository checkout
     """
     if len(changes) == 1:
         slug = f"update-{changes[0][0]}"
@@ -244,7 +233,7 @@ def create_releasenote(changes):
         slug = "update-external-components"
 
     result = subprocess.run(  # nosec
-        ['reno', 'new', slug, REPO_ROOT],
+        ['reno', 'new', slug, repo_root],
         capture_output=True, text=True)
 
     if result.returncode != 0:
@@ -269,8 +258,13 @@ def create_releasenote(changes):
     print(f"Created release note: {note_path}")
 
 
-def check_versions(update=False):
-    """Monitor Kolla architectures and compare with latest releases."""
+def check_versions(sources, sources_path, repo_root, update=False):
+    """Monitor Kolla architectures and compare with latest releases.
+
+    :param sources: the kolla.common.sources.SOURCES dict
+    :param sources_path: path to the sources.yaml file to update
+    :param repo_root: root of the kolla repository checkout
+    """
     check_github_rate_limit()
     arch_map = {'amd64': 'linux-amd64', 'arm64': 'linux-arm64'}
     version_changes = []
@@ -280,7 +274,7 @@ def check_versions(update=False):
     print(header)
     print("-" * 110)
 
-    for name, info in SOURCES.items():
+    for name, info in sources.items():
         current_v = info.get('version') or info.get('reference', '')
         if not re.match(r'^v?\d+[\.\d]+', current_v or ''):
             current_v = None
@@ -324,23 +318,54 @@ def check_versions(update=False):
                     print(line)
 
                 if update and lines_to_print:
-                    apply_update(name, info, latest_v, new_hashes)
+                    apply_update(name, info, latest_v, new_hashes,
+                                 sources_path)
                     if status_v == "UPDATE":
                         version_changes.append(
                             (name, current_v, latest_v))
 
     if update and version_changes:
-        create_releasenote(version_changes)
+        create_releasenote(version_changes, repo_root)
 
 
-if __name__ == "__main__":
+def parse_args():
     parser = argparse.ArgumentParser(
         description='Check and optionally update Kolla source versions.')
+    parser.add_argument('--repo', default=None,
+                        help='Path to the kolla repository checkout '
+                             '(default: the checkout this script lives in, '
+                             'if any, otherwise the current directory)')
     parser.add_argument('--update', action='store_true',
-                        help='Update kolla/common/sources.py in place')
-    args = parser.parse_args()
+                        help='Update kolla/common/sources.yaml in place')
+    return parser.parse_args()
+
+
+def default_repo_root():
+    """Guess the repo root when run in-tree from <repo>/tools/."""
+    in_tree = os.path.normpath(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+    if os.path.isfile(
+            os.path.join(in_tree, 'kolla', 'common', 'sources.yaml')):
+        return in_tree
+    return os.getcwd()
+
+
+if __name__ == "__main__":
+    args = parse_args()
     if args.update and not shutil.which('reno'):
         print("ERROR: 'reno' is not installed or not in PATH. "
               "Install it with: pip install reno")
         sys.exit(1)
-    check_versions(update=args.update)
+
+    repo_root = os.path.abspath(args.repo or default_repo_root())
+    sources_path = os.path.join(
+        repo_root, 'kolla', 'common', 'sources.yaml')
+    if not os.path.isfile(sources_path):
+        print(f"ERROR: {sources_path} not found. Pass --repo pointing at "
+              "a kolla repository checkout.")
+        sys.exit(1)
+
+    with open(sources_path) as f:
+        SOURCES = yaml.safe_load(f)
+
+    check_versions(SOURCES, sources_path, repo_root, update=args.update)
diff --git a/tools/validate-build-sources-overrides.py b/tools/validate-build-sources-overrides.py
index 212c380c2..5d9db62f5 100644
--- a/tools/validate-build-sources-overrides.py
+++ b/tools/validate-build-sources-overrides.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Check that every buildable-from-source entry in kolla/common/sources.py
+"""Check that every buildable-from-source entry in kolla/common/sources.yaml
 has a matching CI override in
 roles/kolla-build-config/defaults/main.yml (kolla_build_sources), that
 each such project is declared in the kolla-base job's required-projects
-in zuul.d/base.yaml, and that any pinned git reference in sources.py
+in zuul.d/base.yaml, and that any pinned git reference in sources.yaml
 matches the override-checkout used to fetch it in CI. This ensures CI
 builds every source from a Zuul-cached git checkout instead of
 downloading a tarball from tarballs.opendev.org.
@@ -116,12 +116,12 @@ def main():
             if actual is None:
                 ref_errors.append(
                     f"{project} ({section}) is pinned to reference "
-                    f"'{expected_ref}' in sources.py but has no "
+                    f"'{expected_ref}' in sources.yaml but has no "
                     "override-checkout in zuul.d/base.yaml "
                     "required-projects")
             elif actual != expected_ref:
                 ref_errors.append(
-                    f"{project} ({section}): sources.py reference "
+                    f"{project} ({section}): sources.yaml reference "
                     f"'{expected_ref}' != required-projects "
                     f"override-checkout '{actual}'")
 
@@ -135,23 +135,23 @@ def main():
             ref_errors.append(
                 f"{project} has override-checkout '{checkout}' in "
                 "required-projects but none of its kolla_build_sources "
-                "sections are type 'git' in sources.py")
+                "sections are type 'git' in sources.yaml")
 
     if missing:
-        print("ERROR: sources.py entries missing a kolla_build_sources "
+        print("ERROR: sources.yaml entries missing a kolla_build_sources "
               "override in roles/kolla-build-config/defaults/main.yml:")
         for name in missing:
             print(f"  {name}")
 
     if stale:
         print("ERROR: kolla_build_sources overrides referencing sections "
-              "that no longer exist in kolla/common/sources.py:")
+              "that no longer exist in kolla/common/sources.yaml:")
         for name in stale:
             print(f"  {name}")
 
     if redundant_exclusions:
         print("ERROR: EXCLUDED_SOURCES entries referencing sections that "
-              "no longer exist in kolla/common/sources.py, remove them:")
+              "no longer exist in kolla/common/sources.yaml, remove them:")
         for name in redundant_exclusions:
             print(f"  {name}")
 
@@ -162,7 +162,7 @@ def main():
             print(f"  {name}")
 
     if ref_errors:
-        print("ERROR: git reference mismatches between sources.py and "
+        print("ERROR: git reference mismatches between sources.yaml and "
               "zuul.d/base.yaml required-projects:")
         for msg in ref_errors:
             print(f"  {msg}")