Enable repos only when needed

Disable external repositories by default and enable only when needed.

Depends-on: https://review.opendev.org/696480

Implements: blueprint repos-off-by-default

Change-Id: Icf2a8397a8349e0fe849d88d160409fd234480a9

4 files changed, 133 insertions, 0 deletions

diff --git a/kolla/image/build.py b/kolla/image/build.py
index 1ad51ee35..34b3dde7f 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -921,6 +921,7 @@ class KollaWorker(object):
 
         return {
             'debian_package_install': jinja_methods.debian_package_install,
+            'enable_repos': jinja_methods.enable_repos,
         }
 
     def get_users(self):
diff --git a/kolla/template/methods.py b/kolla/template/methods.py
index 017b219c7..1217547bf 100644
--- a/kolla/template/methods.py
+++ b/kolla/template/methods.py
@@ -12,6 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import yaml
+
+from jinja2 import contextfunction
+from jinja2.runtime import Undefined
+
 
 def debian_package_install(packages, clean_package_cache=True):
     """Jinja utility method for building debian-based package install command.
@@ -64,3 +70,60 @@ def debian_package_install(packages, clean_package_cache=True):
 
     # return the list of commands
     return ' && '.join(cmds)
+
+
+@contextfunction
+def enable_repos(context, reponames):
+    """NOTE(hrw): we need to handle CentOS, Debian and Ubuntu with one macro.
+
+    Repo names have to be simple names mapped to proper ones.  So 'ceph' ==
+    'centos-ceph-nautilus' for CentOS, UCA for Ubuntu (enabled by default) and
+    something else for Debian.
+    """
+    repofile = os.path.dirname(os.path.realpath(__file__)) + '/repos.yaml'
+    with open(repofile, 'r') as repos_file:
+        repo_data = {}
+        for name, params in yaml.safe_load(repos_file).items():
+            repo_data[name] = params
+
+    # TODO(hrw): add checks for isinstance() and raise proper exception
+    base_package_type = context.get('base_package_type')
+    if isinstance(base_package_type, Undefined):
+        raise
+
+    base_distro = context.get('base_distro')
+    base_arch = context.get('base_arch')
+    distro_package_manager = context.get('distro_package_manager')
+
+    commands = ''
+
+    if base_package_type == 'rpm':
+        # NOTE(hrw): we enable all repos with one call
+        if distro_package_manager == 'yum':
+            commands = 'yum-config-manager '
+        elif distro_package_manager == 'dnf':
+            commands = 'dnf config-manager '
+
+    try:
+        repo_list = repo_data['%s-%s' % (base_distro, base_arch)]
+    except KeyError:
+        # NOTE(hrw): Fallback to distro list
+        repo_list = repo_data[base_distro]
+
+    for repo in reponames:
+        try:
+            if base_package_type == 'rpm':
+                commands += ' --enable %s' % repo_list[repo]
+            elif base_package_type == 'deb':
+                commands += 'echo "%s" ' % repo_list[repo]
+                commands += '>/etc/apt/sources.list.d/%s.list; ' % repo
+        except KeyError:
+            pass
+        # NOTE(hrw): tripleo builds have empty repolist
+        except TypeError:
+            pass
+
+    if commands:
+        commands = "RUN %s" % commands
+
+    return commands
diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml
new file mode 100644
index 000000000..7bfdb0d63
--- /dev/null
+++ b/kolla/template/repos.yaml
@@ -0,0 +1,57 @@
+---
+centos:
+  ceph: "centos-ceph-nautilus"
+  elasticsearch: "elasticsearch-kibana-logstash-5.x"
+  grafana: "grafana"
+  hacluster: "network_ha-clustering_Stable"
+  influxdb: "influxdb"
+  kibana: "elasticsearch-kibana-logstash-5.x"
+  nfs-ganesha: "centos-nfs-ganesha28"
+  opendaylight: "opendaylight"
+  rabbitmq: "rabbitmq_rabbitmq-server"
+  td-agent: "treasuredata"
+
+centos-aarch64:
+  ceph: "centos-ceph-nautilus"
+  elasticsearch: "elasticsearch-kibana-logstash-5.x"
+  grafana: "grafana"
+  nfs-ganesha: "centos-nfs-ganesha28"
+  opendaylight: "opendaylight"
+  rabbitmq: "rabbitmq_rabbitmq-server"
+
+centos-ppc64le:
+  ceph: "centos-ceph-nautilus"
+  elasticsearch: "elasticsearch-kibana-logstash-5.x"
+  nfs-ganesha: "centos-nfs-ganesha28"
+  rabbitmq: "rabbitmq_rabbitmq-server"
+
+rhel:
+
+debian:
+  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  grafana: "deb https://packages.grafana.com/oss/deb stable main"
+  kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib"
+
+ubuntu:
+  elasticsearch: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
+  grafana: "deb https://packages.grafana.com/oss/deb stable main"
+  influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
+  kibana: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
+  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
+  opendaylight: "deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main"
+  qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ bionic main"
+  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main"
+  sensu: "deb https://sensu.global.ssl.fastly.net/apt bionic main"
+  td-agent: "deb http://packages.treasuredata.com/3/ubuntu/bionic/ bionic contrib"
+
+ubuntu-aarch64:
+  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  grafana: "deb https://packages.grafana.com/oss/deb stable main"
+  kibana: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
+  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
+  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main"
+
+ubuntu-ppc64le:
+  mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
+  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main"
diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py
index 1fb5dfa19..62df2f785 100644
--- a/kolla/tests/test_methods.py
+++ b/kolla/tests/test_methods.py
@@ -21,3 +21,15 @@ class MethodsTest(base.TestCase):
         result = methods.debian_package_install(packages)
         expectCmd = 'apt-get -y install --no-install-recommends package2.deb'
         self.assertEqual(expectCmd, result.split("&&")[1].strip())
+
+    def test_enable_repos(self):
+        template_vars = {
+            'base_arch': 'x86_64',
+            'base_distro': 'centos',
+            'base_package_type': 'rpm',
+            'distro_package_manager': 'yum'
+        }
+
+        result = methods.enable_repos(template_vars, ['grafana'])
+        expectCmd = 'RUN yum-config-manager  --enable grafana'
+        self.assertEqual(expectCmd, result)