diff options
Diffstat (limited to 'tests')
| -rwxr-xr-x | tests/files/process_build_logs.py | 80 | ||||
| -rw-r--r-- | tests/playbooks/post.yml | 11 | ||||
| -rw-r--r-- | tests/test_kolla_package_report.py | 174 |
3 files changed, 181 insertions, 84 deletions
diff --git a/tests/files/process_build_logs.py b/tests/files/process_build_logs.py deleted file mode 100755 index 248acf954..000000000 --- a/tests/files/process_build_logs.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/python3 - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import collections -import glob -import os -import re -import sys - -parser = argparse.ArgumentParser( - description='Parse kolla build logs and extract useful information about ' - 'the installed packages.') -parser.add_argument('-l', '--logdir', - help='Path to the build log files', - required=True) -parser.add_argument('-b', '--base', - help='The kolla base_distro', - required=True) -args = vars(parser.parse_args()) - -if args['base'] != 'rocky': - print("Non rpm-based distros are not yet supported.") - sys.exit() - -obsolete = {} -pkg_installs = collections.defaultdict(set) - -for filename in glob.glob(os.path.join(args['logdir'], '*.log')): - image = os.path.splitext(os.path.basename(filename))[0] - with open(filename) as f: - for line in f: - m = re.search(r"Package (.+) is obsoleted by (.+),", line) - if m: - if not m.group(1) in obsolete: - obsolete[m.group(1)] = {'obsoleted_by': m.group(2), - 'images': [image]} - else: - obsolete[m.group(1)]['images'].append(image) - - m = re.search(r"Package (.+)\..+ .+ will be installed", line) - if m: - pkg_installs[m.group(1)].add(image) - - m = re.search(r"Processing Dependency: (.+)\(", line) - if m: - pkg_installs[m.group(1)].add(image) - -if obsolete: - print("Found %s obsolete(s) package(s):" % len(obsolete)) - for pkg in obsolete: - print("%s is obsoleted by %s (%s)" % - (pkg, - obsolete[pkg]['obsoleted_by'], - ', '.join(obsolete[pkg]['images']))) - print('') - -move_to_base_candidates = [ - pkg for pkg in pkg_installs if len(pkg_installs[pkg]) > 10 - and not ('base' in pkg_installs[pkg] - or 'openstack-base' in pkg_installs[pkg]) -] - -if move_to_base_candidates: - print("Consider moving the following packages to a base image:") - for pkg in move_to_base_candidates: - print("%s (%s)" % - (pkg, - ', '.join(pkg_installs[pkg]))) diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml index a5998468b..028fc7b5e 100644 --- a/tests/playbooks/post.yml +++ b/tests/playbooks/post.yml @@ -68,9 +68,12 @@ # last sudo to own all files sudo chown -R {{ ansible_user_id }}: . - - # Parse build logs and extract pkg install info - {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py \ - -l build -b {{ base_distro }} > packages-info.txt executable: /bin/bash chdir: "{{ kolla_logs_dir }}" + + - name: Import kolla-build-tests role + import_role: + name: kolla-build-tests + vars: + kolla_build_tests_logs_dir: "{{ kolla_build_logs_dir }}" + kolla_build_tests_report_dir: "{{ kolla_logs_dir }}" diff --git a/tests/test_kolla_package_report.py b/tests/test_kolla_package_report.py new file mode 100644 index 000000000..61618fa3e --- /dev/null +++ b/tests/test_kolla_package_report.py @@ -0,0 +1,174 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import os.path +import sys + +import fixtures +from oslotest import base + + +def load_module(name, path): + module_spec = importlib.util.spec_from_file_location( + name, path + ) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + return module + + +# kolla_package_report is an Ansible module in a role, not part of the kolla +# package, so load it by path. +this_dir = os.path.dirname(sys.modules[__name__].__file__) +kolla_package_report = load_module( + 'kolla_package_report', + os.path.join(this_dir, '..', 'roles', 'kolla-build-tests', 'library', + 'kolla_package_report.py')) + + +class ParseParentTest(base.BaseTestCase): + + def test_buildkit(self): + self.assertEqual('nova-base', kolla_package_report.parse_parent([ + 'INFO:kolla.common.utils.nova-api:#2 [internal] load metadata ' + 'for docker.io/kolla/nova-base:master-rocky-10\n', + 'INFO:kolla.common.utils.nova-api:#4 [1/7] FROM ' + 'docker.io/kolla/nova-base:master-rocky-10@sha256:00\n'])) + + def test_buildkit_padded(self): + self.assertEqual('openstack-base', kolla_package_report.parse_parent([ + 'INFO:kolla.common.utils.nova-base:#5 [ 1/23] FROM ' + 'docker.io/kolla/openstack-base:master-rocky-10@sha256:00\n'])) + + def test_podman(self): + self.assertEqual('openstack-base', kolla_package_report.parse_parent([ + 'INFO:kolla.common.utils.nova-base:STEP 1/15: FROM ' + 'kolla/openstack-base:master-rocky-10\n'])) + + def test_external_base(self): + self.assertEqual('rockylinux', kolla_package_report.parse_parent([ + 'INFO:kolla.common.utils.base:#5 [ 1/23] FROM ' + 'quay.io/rockylinux/rockylinux:10@sha256:00\n'])) + + def test_no_from(self): + self.assertIsNone(kolla_package_report.parse_parent([ + 'INFO:kolla.common.utils.base:Building started\n', + 'INFO:kolla.common.utils.base:#5 ' + 'x' * 200000 + '\n'])) + + +class AnalyseTest(base.BaseTestCase): + + PARENTS = {'base': None, 'openstack-base': 'base', + 'nova-base': 'openstack-base', 'nova-api': 'nova-base', + 'nova-compute': 'nova-base', 'cinder-base': 'openstack-base'} + + def setUp(self): + super().setUp() + tmp = self.useFixture(fixtures.TempDir()).path + self.logdir = os.path.join(tmp, 'build') + self.pkgdir = os.path.join(tmp, 'packages') + os.mkdir(self.logdir) + os.mkdir(self.pkgdir) + + def _image(self, image, parent, packages=None): + with open(os.path.join(self.logdir, image + '.log'), 'w') as f: + f.write('INFO:kolla.common.utils.%s:#4 [1/3] FROM ' + 'docker.io/kolla/%s:master-rocky-10@sha256:00\n' % + (image, parent)) + if packages is not None: + with open(os.path.join(self.pkgdir, image + '.txt'), 'w') as f: + f.write(''.join(p + '\n' for p in packages)) + + def test_common_parent(self): + self.assertEqual('nova-base', kolla_package_report.common_parent( + ['nova-api', 'nova-compute'], self.PARENTS)) + self.assertEqual('openstack-base', kolla_package_report.common_parent( + ['nova-api', 'cinder-base'], self.PARENTS)) + self.assertIsNone(kolla_package_report.common_parent( + ['base', 'nova-api'], self.PARENTS)) + + def test_analyse(self): + self._image('base', 'rockylinux', ['bash', 'libbar', 'gpg-pubkey']) + self._image('openstack-base', 'base', ['bash', 'git']) + self._image('nova-base', 'openstack-base', ['bash', 'git', 'libnl3']) + self._image('cinder-base', 'openstack-base', ['bash', 'git', 'libnl3']) + self._image('nova-api', 'nova-base', + ['bash', 'git', 'libnl3', 'libbar', 'libfoo']) + self._image('nova-compute', 'nova-base', + ['bash', 'git', 'libnl3', 'libfoo', 'gpg-pubkey']) + + report = kolla_package_report.analyse(self.logdir, self.pkgdir, 1) + + self.assertEqual( + [{'package': 'libbar', + 'images': ['base', 'nova-api'], + 'suggested_parent': None}, + {'package': 'libfoo', + 'images': ['nova-api', 'nova-compute'], + 'suggested_parent': 'nova-base'}, + {'package': 'libnl3', + 'images': ['cinder-base', 'nova-base'], + 'suggested_parent': 'openstack-base'}], + report['candidates']) + self.assertEqual( + [{'image': 'openstack-base', 'parent': 'base', + 'packages': ['libbar']}], + report['removals']) + self.assertEqual([], report['skipped']) + self.assertEqual( + ['libbar is installed by 2 images separately, but they have no ' + 'common parent image', + 'libfoo is installed by 2 images separately; consider ' + 'installing it once in nova-base instead', + 'libnl3 is installed by 2 images separately; consider ' + 'installing it once in openstack-base instead', + 'openstack-base removes libbar, which its parent base has ' + 'installed'], + report['warnings']) + self.assertEqual([], kolla_package_report.analyse( + self.logdir, self.pkgdir, 2)['candidates']) + + def test_parent_without_package_list_skipped(self): + self._image('base', 'rockylinux', ['bash']) + self._image('openstack-base', 'base') + self._image('nova-base', 'openstack-base', ['bash', 'git', 'libnl3']) + + report = kolla_package_report.analyse(self.logdir, self.pkgdir, 0) + + self.assertEqual(['nova-base'], report['skipped']) + self.assertEqual(['bash'], + [c['package'] for c in report['candidates']]) + + def test_format_report(self): + report = {'candidates': [{'package': 'libnl3', + 'images': ['cinder-base', 'nova-base'], + 'suggested_parent': 'openstack-base'}, + {'package': 'libbar', + 'images': ['base', 'nova-api'], + 'suggested_parent': None}], + 'removals': [{'image': 'cinder-volume', + 'parent': 'cinder-base', + 'packages': ['foo', 'bar']}], + 'skipped': ['nova-base']} + self.assertEqual( + 'Consider moving the following packages to a parent image:\n' + 'libnl3 -> openstack-base (cinder-base, nova-base)\n' + 'libbar -> (no common parent) (base, nova-api)\n' + '\n' + 'Packages removed compared with the parent image:\n' + 'cinder-volume (parent cinder-base): foo, bar\n' + '\n' + 'Skipped (parent has no package list): nova-base\n', + kolla_package_report.format_report(report)) + self.assertEqual('', kolla_package_report.format_report( + {'candidates': [], 'removals': [], 'skipped': []})) |
