Add support to override base repos

BaseOS and AppStream for CentOS/Rocky
ubuntu, ubuntu-security for Ubuntu
debian, debian-security for Debian

ubuntu-cloud-archive is not really a base repo - so
removing it from ubuntu.sources

Change-Id: Iab98de963f2e8532389b7563bb572d8aa20c7aa6
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>

1 files changed, 198 insertions, 14 deletions

diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py
index e1dc810d1..8a577aead 100644
--- a/kolla/tests/test_methods.py
+++ b/kolla/tests/test_methods.py
@@ -10,6 +10,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import tempfile
+
+import yaml
+
 from kolla.template import methods
 from kolla.tests import base
 
@@ -30,7 +34,9 @@ class MethodsTest(base.TestCase):
         }
 
         result = methods.handle_repos(template_vars, ["grafana"], "enable")
-        expectCmd = "RUN echo '[grafana]' "
+        expectCmd = "RUN grep -rlF '[grafana]' /etc/yum.repos.d/ "
+        expectCmd += "2>/dev/null | xargs -r rm -f && "
+        expectCmd += "echo '[grafana]' "
         expectCmd += ">/etc/yum.repos.d/grafana.repo && "
         expectCmd += "echo 'name=grafana' "
         expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
@@ -42,7 +48,7 @@ class MethodsTest(base.TestCase):
         expectCmd += ">>/etc/yum.repos.d/grafana.repo"
         self.assertEqual(expectCmd, result)
 
-    def test_enable_repos_centos_metalink(self):
+    def test_enable_repos_centos_ceph_distro(self):
         template_vars = {
             'base_arch': 'x86_64',
             'base_distro': 'centos',
@@ -50,16 +56,7 @@ class MethodsTest(base.TestCase):
         }
 
         result = methods.handle_repos(template_vars, ["ceph"], "enable")
-        expectCmd = "RUN echo '[centos-ceph-squid]' "
-        expectCmd += ">/etc/yum.repos.d/ceph.repo && "
-        expectCmd += "echo 'name=centos-ceph-squid' "
-        expectCmd += ">>/etc/yum.repos.d/ceph.repo && "
-        expectCmd += "echo 'enabled=1' "
-        expectCmd += ">>/etc/yum.repos.d/ceph.repo && "
-        expectCmd += "echo 'gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-Storage' "  # noqa: E501
-        expectCmd += ">>/etc/yum.repos.d/ceph.repo && "
-        expectCmd += "echo 'metalink=https://mirrors.centos.org/metalink?repo=centos-storage-sig-ceph-squid-10-stream&arch=$basearch' "  # noqa: E501
-        expectCmd += ">>/etc/yum.repos.d/ceph.repo"
+        expectCmd = "RUN dnf config-manager --enable centos-ceph-squid || true"
         self.assertEqual(expectCmd, result)
 
     def test_enable_repos_centos_arch(self):
@@ -70,7 +67,9 @@ class MethodsTest(base.TestCase):
         }
 
         result = methods.handle_repos(template_vars, ["grafana"], "enable")
-        expectCmd = "RUN echo '[grafana]' "
+        expectCmd = "RUN grep -rlF '[grafana]' /etc/yum.repos.d/ "
+        expectCmd += "2>/dev/null | xargs -r rm -f && "
+        expectCmd += "echo '[grafana]' "
         expectCmd += ">/etc/yum.repos.d/grafana.repo && "
         expectCmd += "echo 'name=grafana' "
         expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
@@ -91,7 +90,9 @@ class MethodsTest(base.TestCase):
 
         result = methods.handle_repos(template_vars,
                                       ["grafana", "rabbitmq"], "enable")
-        expectCmd = "RUN echo '[grafana]' "
+        expectCmd = "RUN grep -rlF '[grafana]' /etc/yum.repos.d/ "
+        expectCmd += "2>/dev/null | xargs -r rm -f && "
+        expectCmd += "echo '[grafana]' "
         expectCmd += ">/etc/yum.repos.d/grafana.repo && "
         expectCmd += "echo 'name=grafana' "
         expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
@@ -102,6 +103,8 @@ class MethodsTest(base.TestCase):
         expectCmd += "echo 'baseurl=https://rpm.grafana.com' "
         expectCmd += ">>/etc/yum.repos.d/grafana.repo && "
 
+        expectCmd += "grep -rlF '[rabbitmq_rabbitmq-server]' "
+        expectCmd += "/etc/yum.repos.d/ 2>/dev/null | xargs -r rm -f && "
         expectCmd += "echo '[rabbitmq_rabbitmq-server]' "
         expectCmd += ">/etc/yum.repos.d/rabbitmq.repo && "
         expectCmd += "echo 'name=rabbitmq_rabbitmq-server' "
@@ -151,6 +154,8 @@ class MethodsTest(base.TestCase):
         result = methods.handle_repos(template_vars,
                                       ['crb', 'grafana'], 'enable')
         expectCmd = "RUN dnf config-manager --enable crb || true && "
+        expectCmd += "grep -rlF '[grafana]' /etc/yum.repos.d/ "
+        expectCmd += "2>/dev/null | xargs -r rm -f && "
         expectCmd += "echo '[grafana]' "
         expectCmd += ">/etc/yum.repos.d/grafana.repo && "
         expectCmd += "echo 'name=grafana' "
@@ -168,6 +173,7 @@ class MethodsTest(base.TestCase):
             "base_arch": "x86_64",
             "base_distro": "debian",
             "base_package_type": "deb",
+            "openstack_release_codename": "Gazpacho",
         }
 
         result = methods.handle_repos(template_vars, ["grafana"], "enable")
@@ -188,6 +194,7 @@ class MethodsTest(base.TestCase):
             'base_arch': 'aarch64',
             'base_distro': 'debian',
             'base_package_type': 'deb',
+            'openstack_release_codename': 'Gazpacho',
         }
 
         result = methods.handle_repos(template_vars, ["rabbitmq"], "enable")
@@ -220,6 +227,7 @@ class MethodsTest(base.TestCase):
             'base_arch': 'x86_64',
             'base_distro': 'debian',
             'base_package_type': 'deb',
+            'openstack_release_codename': 'Gazpacho',
         }
 
         result = methods.handle_repos(template_vars,
@@ -273,3 +281,179 @@ class MethodsTest(base.TestCase):
                                r'repositories',
                                methods.handle_repos, template_vars, 'grafana',
                                'disable')
+
+    def test_enable_repos_debian_distro_noop(self):
+        template_vars = {
+            'base_arch': 'x86_64',
+            'base_distro': 'debian',
+            'base_package_type': 'deb',
+        }
+
+        result = methods.handle_repos(template_vars, ['debian'], 'enable')
+        self.assertEqual('', result)
+
+    def test_enable_repos_deb_absolute_signed_by(self):
+        repos = {'debian': {'test-repo': {
+            'url': 'http://example.com/debian',
+            'suite': 'trixie',
+            'component': 'main',
+            'gpg_key': '/usr/share/keyrings/test.gpg',
+        }}}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'debian',
+                'base_package_type': 'deb',
+                'openstack_release_codename': 'Gazpacho',
+                'repos_yaml': f.name,
+            }
+            result = methods.handle_repos(
+                template_vars, ['test-repo'], 'enable')
+            self.assertIn(
+                'Signed-By: /usr/share/keyrings/test.gpg', result)
+
+    def test_repos_yaml_merge_keeps_default_repos(self):
+        repos = {'rocky': {'baseos': {
+            'name': 'baseos',
+            'baseurl': 'http://mirror.example.com/rocky/10/BaseOS/$basearch/',
+            'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10',
+        }}}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'centos',
+                'base_package_type': 'rpm',
+                'repos_yaml': f.name,
+            }
+            expected_commands = methods.handle_repos(
+                template_vars, ['grafana'], 'enable')
+            self.assertIn(
+                "echo 'gpgkey=https://rpm.grafana.com/gpg.key' "
+                ">>/etc/yum.repos.d/grafana.repo",
+                expected_commands)
+
+    def test_repos_yaml_distro_override_validation(self):
+        """Overriding baseos and appstream would delete rocky.repo.
+
+        crb is a distro repo that also uses that file group. Verify a
+        ValueError is raised so the user knows to override crb as well.
+        """
+        repos = {'rocky': {
+            'baseos': {
+                'name': 'baseos',
+                'baseurl': 'http://mirror.example.com/rocky/10/BaseOS/',
+                'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10',
+            },
+            'appstream': {
+                'name': 'appstream',
+                'baseurl': 'http://mirror.example.com/rocky/10/AppStream/',
+                'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-10',
+            },
+        }}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'rocky',
+                'base_package_type': 'rpm',
+                'repos_yaml': f.name,
+            }
+            self.assertRaisesRegex(
+                ValueError, "still using distro defaults and share the same",
+                methods.handle_repos, template_vars, ['baseos'], 'enable')
+
+    def test_enable_repos_rpm_missing_url_raises(self):
+        repos = {'rpm': {'test-repo': {'name': 'test-repo'}}}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'centos',
+                'base_package_type': 'rpm',
+                'repos_yaml': f.name,
+            }
+            self.assertRaises(ValueError, methods.handle_repos,
+                              template_vars, ['test-repo'], 'enable')
+
+    def test_enable_repos_rpm_missing_gpgkey_raises(self):
+        repos = {'rpm': {'test-repo': {
+            'name': 'test-repo',
+            'baseurl': 'http://mirror.example.com/repo/',
+        }}}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'centos',
+                'base_package_type': 'rpm',
+                'repos_yaml': f.name,
+            }
+            self.assertRaises(ValueError, methods.handle_repos,
+                              template_vars, ['test-repo'], 'enable')
+
+    def test_repos_yaml_rpm_section_override_not_blocked_by_distro_section(
+            self):
+        """Overriding distro repos in 'rpm' section must not be undone.
+
+        The more-specific distro section must not merge distro:True back
+        on top of a URL-bearing override from the generic 'rpm' section.
+        """
+        base = 'http://mirror.example.com/centos/10'
+        gpgkey = 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial'
+        repos = {'rpm': {
+            'baseos': {
+                'name': 'baseos',
+                'baseurl': base + '/BaseOS/$basearch/',
+                'gpgkey': gpgkey,
+            },
+            'appstream': {
+                'name': 'appstream',
+                'baseurl': base + '/AppStream/$basearch/',
+                'gpgkey': gpgkey,
+            },
+            'crb': {
+                'name': 'crb',
+                'baseurl': base + '/CRB/$basearch/',
+                'gpgkey': gpgkey,
+            },
+        }}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'centos',
+                'base_package_type': 'rpm',
+                'repos_yaml': f.name,
+            }
+            result = methods.handle_repos(
+                template_vars, ['baseos', 'appstream', 'crb'], 'enable')
+            self.assertIn('mirror.example.com', result)
+
+    def test_enable_repos_openstack_release_codename_substitution(self):
+        repos = {'ubuntu': {'ubuntu-cloud-archive': {
+            'url': 'http://ubuntu-cloud.archive.canonical.com/ubuntu',
+            'suite': 'noble-updates/{openstack_release_codename}',
+            'component': 'main',
+            'gpg_key': '/usr/share/keyrings/ubuntu-cloud-keyring.gpg',
+        }}}
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml') as f:
+            yaml.dump(repos, f)
+            f.flush()
+            template_vars = {
+                'base_arch': 'x86_64',
+                'base_distro': 'ubuntu',
+                'base_package_type': 'deb',
+                'openstack_release_codename': 'Gazpacho',
+                'repos_yaml': f.name,
+            }
+            result = methods.handle_repos(
+                template_vars, ['ubuntu-cloud-archive'], 'enable')
+            self.assertIn('noble-updates/gazpacho', result)