aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rwxr-xr-xtests/files/process_build_logs.py80
-rw-r--r--tests/playbooks/post.yml11
-rw-r--r--tests/test_kolla_package_report.py174
3 files changed, 181 insertions, 84 deletions
diff --git a/tests/files/process_build_logs.py b/tests/files/process_build_logs.py
deleted file mode 100755
index 248acf954..000000000
--- a/tests/files/process_build_logs.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/python3
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import collections
-import glob
-import os
-import re
-import sys
-
-parser = argparse.ArgumentParser(
- description='Parse kolla build logs and extract useful information about '
- 'the installed packages.')
-parser.add_argument('-l', '--logdir',
- help='Path to the build log files',
- required=True)
-parser.add_argument('-b', '--base',
- help='The kolla base_distro',
- required=True)
-args = vars(parser.parse_args())
-
-if args['base'] != 'rocky':
- print("Non rpm-based distros are not yet supported.")
- sys.exit()
-
-obsolete = {}
-pkg_installs = collections.defaultdict(set)
-
-for filename in glob.glob(os.path.join(args['logdir'], '*.log')):
- image = os.path.splitext(os.path.basename(filename))[0]
- with open(filename) as f:
- for line in f:
- m = re.search(r"Package (.+) is obsoleted by (.+),", line)
- if m:
- if not m.group(1) in obsolete:
- obsolete[m.group(1)] = {'obsoleted_by': m.group(2),
- 'images': [image]}
- else:
- obsolete[m.group(1)]['images'].append(image)
-
- m = re.search(r"Package (.+)\..+ .+ will be installed", line)
- if m:
- pkg_installs[m.group(1)].add(image)
-
- m = re.search(r"Processing Dependency: (.+)\(", line)
- if m:
- pkg_installs[m.group(1)].add(image)
-
-if obsolete:
- print("Found %s obsolete(s) package(s):" % len(obsolete))
- for pkg in obsolete:
- print("%s is obsoleted by %s (%s)" %
- (pkg,
- obsolete[pkg]['obsoleted_by'],
- ', '.join(obsolete[pkg]['images'])))
- print('')
-
-move_to_base_candidates = [
- pkg for pkg in pkg_installs if len(pkg_installs[pkg]) > 10
- and not ('base' in pkg_installs[pkg]
- or 'openstack-base' in pkg_installs[pkg])
-]
-
-if move_to_base_candidates:
- print("Consider moving the following packages to a base image:")
- for pkg in move_to_base_candidates:
- print("%s (%s)" %
- (pkg,
- ', '.join(pkg_installs[pkg])))
diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml
index a5998468b..028fc7b5e 100644
--- a/tests/playbooks/post.yml
+++ b/tests/playbooks/post.yml
@@ -68,9 +68,12 @@
# last sudo to own all files
sudo chown -R {{ ansible_user_id }}: .
-
- # Parse build logs and extract pkg install info
- {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py \
- -l build -b {{ base_distro }} > packages-info.txt
executable: /bin/bash
chdir: "{{ kolla_logs_dir }}"
+
+ - name: Import kolla-build-tests role
+ import_role:
+ name: kolla-build-tests
+ vars:
+ kolla_build_tests_logs_dir: "{{ kolla_build_logs_dir }}"
+ kolla_build_tests_report_dir: "{{ kolla_logs_dir }}"
diff --git a/tests/test_kolla_package_report.py b/tests/test_kolla_package_report.py
new file mode 100644
index 000000000..61618fa3e
--- /dev/null
+++ b/tests/test_kolla_package_report.py
@@ -0,0 +1,174 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.util
+import os.path
+import sys
+
+import fixtures
+from oslotest import base
+
+
+def load_module(name, path):
+ module_spec = importlib.util.spec_from_file_location(
+ name, path
+ )
+ module = importlib.util.module_from_spec(module_spec)
+ module_spec.loader.exec_module(module)
+ return module
+
+
+# kolla_package_report is an Ansible module in a role, not part of the kolla
+# package, so load it by path.
+this_dir = os.path.dirname(sys.modules[__name__].__file__)
+kolla_package_report = load_module(
+ 'kolla_package_report',
+ os.path.join(this_dir, '..', 'roles', 'kolla-build-tests', 'library',
+ 'kolla_package_report.py'))
+
+
+class ParseParentTest(base.BaseTestCase):
+
+ def test_buildkit(self):
+ self.assertEqual('nova-base', kolla_package_report.parse_parent([
+ 'INFO:kolla.common.utils.nova-api:#2 [internal] load metadata '
+ 'for docker.io/kolla/nova-base:master-rocky-10\n',
+ 'INFO:kolla.common.utils.nova-api:#4 [1/7] FROM '
+ 'docker.io/kolla/nova-base:master-rocky-10@sha256:00\n']))
+
+ def test_buildkit_padded(self):
+ self.assertEqual('openstack-base', kolla_package_report.parse_parent([
+ 'INFO:kolla.common.utils.nova-base:#5 [ 1/23] FROM '
+ 'docker.io/kolla/openstack-base:master-rocky-10@sha256:00\n']))
+
+ def test_podman(self):
+ self.assertEqual('openstack-base', kolla_package_report.parse_parent([
+ 'INFO:kolla.common.utils.nova-base:STEP 1/15: FROM '
+ 'kolla/openstack-base:master-rocky-10\n']))
+
+ def test_external_base(self):
+ self.assertEqual('rockylinux', kolla_package_report.parse_parent([
+ 'INFO:kolla.common.utils.base:#5 [ 1/23] FROM '
+ 'quay.io/rockylinux/rockylinux:10@sha256:00\n']))
+
+ def test_no_from(self):
+ self.assertIsNone(kolla_package_report.parse_parent([
+ 'INFO:kolla.common.utils.base:Building started\n',
+ 'INFO:kolla.common.utils.base:#5 ' + 'x' * 200000 + '\n']))
+
+
+class AnalyseTest(base.BaseTestCase):
+
+ PARENTS = {'base': None, 'openstack-base': 'base',
+ 'nova-base': 'openstack-base', 'nova-api': 'nova-base',
+ 'nova-compute': 'nova-base', 'cinder-base': 'openstack-base'}
+
+ def setUp(self):
+ super().setUp()
+ tmp = self.useFixture(fixtures.TempDir()).path
+ self.logdir = os.path.join(tmp, 'build')
+ self.pkgdir = os.path.join(tmp, 'packages')
+ os.mkdir(self.logdir)
+ os.mkdir(self.pkgdir)
+
+ def _image(self, image, parent, packages=None):
+ with open(os.path.join(self.logdir, image + '.log'), 'w') as f:
+ f.write('INFO:kolla.common.utils.%s:#4 [1/3] FROM '
+ 'docker.io/kolla/%s:master-rocky-10@sha256:00\n' %
+ (image, parent))
+ if packages is not None:
+ with open(os.path.join(self.pkgdir, image + '.txt'), 'w') as f:
+ f.write(''.join(p + '\n' for p in packages))
+
+ def test_common_parent(self):
+ self.assertEqual('nova-base', kolla_package_report.common_parent(
+ ['nova-api', 'nova-compute'], self.PARENTS))
+ self.assertEqual('openstack-base', kolla_package_report.common_parent(
+ ['nova-api', 'cinder-base'], self.PARENTS))
+ self.assertIsNone(kolla_package_report.common_parent(
+ ['base', 'nova-api'], self.PARENTS))
+
+ def test_analyse(self):
+ self._image('base', 'rockylinux', ['bash', 'libbar', 'gpg-pubkey'])
+ self._image('openstack-base', 'base', ['bash', 'git'])
+ self._image('nova-base', 'openstack-base', ['bash', 'git', 'libnl3'])
+ self._image('cinder-base', 'openstack-base', ['bash', 'git', 'libnl3'])
+ self._image('nova-api', 'nova-base',
+ ['bash', 'git', 'libnl3', 'libbar', 'libfoo'])
+ self._image('nova-compute', 'nova-base',
+ ['bash', 'git', 'libnl3', 'libfoo', 'gpg-pubkey'])
+
+ report = kolla_package_report.analyse(self.logdir, self.pkgdir, 1)
+
+ self.assertEqual(
+ [{'package': 'libbar',
+ 'images': ['base', 'nova-api'],
+ 'suggested_parent': None},
+ {'package': 'libfoo',
+ 'images': ['nova-api', 'nova-compute'],
+ 'suggested_parent': 'nova-base'},
+ {'package': 'libnl3',
+ 'images': ['cinder-base', 'nova-base'],
+ 'suggested_parent': 'openstack-base'}],
+ report['candidates'])
+ self.assertEqual(
+ [{'image': 'openstack-base', 'parent': 'base',
+ 'packages': ['libbar']}],
+ report['removals'])
+ self.assertEqual([], report['skipped'])
+ self.assertEqual(
+ ['libbar is installed by 2 images separately, but they have no '
+ 'common parent image',
+ 'libfoo is installed by 2 images separately; consider '
+ 'installing it once in nova-base instead',
+ 'libnl3 is installed by 2 images separately; consider '
+ 'installing it once in openstack-base instead',
+ 'openstack-base removes libbar, which its parent base has '
+ 'installed'],
+ report['warnings'])
+ self.assertEqual([], kolla_package_report.analyse(
+ self.logdir, self.pkgdir, 2)['candidates'])
+
+ def test_parent_without_package_list_skipped(self):
+ self._image('base', 'rockylinux', ['bash'])
+ self._image('openstack-base', 'base')
+ self._image('nova-base', 'openstack-base', ['bash', 'git', 'libnl3'])
+
+ report = kolla_package_report.analyse(self.logdir, self.pkgdir, 0)
+
+ self.assertEqual(['nova-base'], report['skipped'])
+ self.assertEqual(['bash'],
+ [c['package'] for c in report['candidates']])
+
+ def test_format_report(self):
+ report = {'candidates': [{'package': 'libnl3',
+ 'images': ['cinder-base', 'nova-base'],
+ 'suggested_parent': 'openstack-base'},
+ {'package': 'libbar',
+ 'images': ['base', 'nova-api'],
+ 'suggested_parent': None}],
+ 'removals': [{'image': 'cinder-volume',
+ 'parent': 'cinder-base',
+ 'packages': ['foo', 'bar']}],
+ 'skipped': ['nova-base']}
+ self.assertEqual(
+ 'Consider moving the following packages to a parent image:\n'
+ 'libnl3 -> openstack-base (cinder-base, nova-base)\n'
+ 'libbar -> (no common parent) (base, nova-api)\n'
+ '\n'
+ 'Packages removed compared with the parent image:\n'
+ 'cinder-volume (parent cinder-base): foo, bar\n'
+ '\n'
+ 'Skipped (parent has no package list): nova-base\n',
+ kolla_package_report.format_report(report))
+ self.assertEqual('', kolla_package_report.format_report(
+ {'candidates': [], 'removals': [], 'skipped': []}))