diff options
| -rwxr-xr-x | kolla/cmd/build.py | 2 | ||||
| -rwxr-xr-x | kolla/image/build.py | 127 | ||||
| -rw-r--r-- | kolla/tests/test_build.py | 15 | ||||
| -rw-r--r-- | tests/test_build.py | 4 |
4 files changed, 105 insertions, 43 deletions
diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py index e912a6d12..3ccc4475a 100755 --- a/kolla/cmd/build.py +++ b/kolla/cmd/build.py @@ -33,7 +33,7 @@ def main(): statuses = build.run_build() if statuses: (bad_results, good_results, unmatched_results, - skipped_results) = statuses + skipped_results, unbuildable_results) = statuses if bad_results: return 1 return 0 diff --git a/kolla/image/build.py b/kolla/image/build.py index d5c51d73b..7b8dcd9af 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -71,16 +71,17 @@ STATUS_UNMATCHED = 'unmatched' STATUS_MATCHED = 'matched' STATUS_UNPROCESSED = 'unprocessed' STATUS_SKIPPED = 'skipped' +STATUS_UNBUILDABLE = 'unbuildable' # All error status constants. STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, STATUS_ERROR, STATUS_PARENT_ERROR) -# The dictionary of skipped images supports keys in the format: +# The dictionary of unbuildable images supports keys in the format: # '<distro>+<installation_type>+<arch>' where each component is optional # and can be omitted along with the + separator which means that component # is irrelevant. Otherwise all must match for skip to happen. -SKIPPED_IMAGES = { +UNBUILDABLE_IMAGES = { 'aarch64': { "cyborg-base", # no binary package "kibana", # no binary package @@ -505,7 +506,7 @@ class BuildTask(DockerTask): self.logger.debug('Processing') - if image.status == STATUS_SKIPPED: + if image.status in [STATUS_SKIPPED, STATUS_UNBUILDABLE]: self.logger.info('Skipping %s' % image.name) return @@ -741,6 +742,7 @@ class KollaWorker(object): self.image_statuses_good = dict() self.image_statuses_unmatched = dict() self.image_statuses_skipped = dict() + self.image_statuses_unbuildable = dict() self.maintainer = conf.maintainer self.distro_python_version = conf.distro_python_version @@ -1000,44 +1002,80 @@ class KollaWorker(object): else: filter_ += self.conf.profiles[profile] + # mark unbuildable images and their children + tag_element = r'(%s|%s|%s)' % (self.base, + self.install_type, + self.base_arch) + tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element)) + unbuildable_images = set() + for set_tag in UNBUILDABLE_IMAGES: + if tag_re.match(set_tag): + unbuildable_images.update(UNBUILDABLE_IMAGES[set_tag]) + + if unbuildable_images: + for image in self.images: + if image.name in unbuildable_images: + image.status = STATUS_UNBUILDABLE + else: + # let's check ancestors + # if any of them is unbuildable then we mark it + # and then mark image + build_image = True + ancestor_image = image + while (ancestor_image.parent is not None): + ancestor_image = ancestor_image.parent + if ancestor_image.name in unbuildable_images or \ + ancestor_image.status == STATUS_UNBUILDABLE: + build_image = False + ancestor_image.status = STATUS_UNBUILDABLE + break + if not build_image: + image.status = STATUS_UNBUILDABLE + + # When we want to build a subset of images then filter_ part kicks in. + # Otherwise we just mark everything buildable as matched for build. + if filter_: patterns = re.compile(r"|".join(filter_).join('()')) for image in self.images: - if image.status in (STATUS_MATCHED, STATUS_SKIPPED): + # as we now list not buildable/skipped images we need to + # process them otherwise list will contain also not requested + # entries + if image.status == STATUS_MATCHED: continue if re.search(patterns, image.name): - image.status = STATUS_MATCHED - while (image.parent is not None and - image.parent.status not in (STATUS_MATCHED, - STATUS_SKIPPED)): - image = image.parent + if image.status not in [STATUS_SKIPPED, + STATUS_UNBUILDABLE]: + image.status = STATUS_MATCHED + + # skip image if --skip-existing was given and image + # was already built + if (self.conf.skip_existing and image.in_docker_cache()): + image.status = STATUS_SKIPPED + + # handle image ancestors + ancestor_image = image + while (ancestor_image.parent is not None and + ancestor_image.parent.status not in + (STATUS_MATCHED, STATUS_SKIPPED)): + ancestor_image = ancestor_image.parent if self.conf.skip_parents: - image.status = STATUS_SKIPPED + ancestor_image.status = STATUS_SKIPPED elif (self.conf.skip_existing and - image.in_docker_cache()): - image.status = STATUS_SKIPPED + ancestor_image.in_docker_cache()): + ancestor_image.status = STATUS_SKIPPED else: - image.status = STATUS_MATCHED + if ancestor_image.status != STATUS_UNBUILDABLE: + ancestor_image.status = STATUS_MATCHED LOG.debug('Image %s matched regex', image.name) else: + # we do not care if it is skipped or not as we did not + # request it image.status = STATUS_UNMATCHED else: for image in self.images: - image.status = STATUS_MATCHED - - # Unmatch (skip) unsupported images - tag_element = r'(%s|%s|%s)' % (self.base, - self.install_type, - self.base_arch) - tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element)) - skipped_images = set() - for set_tag in SKIPPED_IMAGES: - if tag_re.match(set_tag): - skipped_images.update(SKIPPED_IMAGES[set_tag]) - if skipped_images: - for image in self.images: - if image.name in skipped_images: - image.status = STATUS_UNMATCHED + if image.status != STATUS_UNBUILDABLE: + image.status = STATUS_MATCHED def summary(self): """Walk the dictionary of images statuses and print results.""" @@ -1053,6 +1091,7 @@ class KollaWorker(object): 'failed': [], 'not_matched': [], 'skipped': [], + 'unbuildable': [], } if self.image_statuses_good: @@ -1097,26 +1136,38 @@ class KollaWorker(object): }) if self.image_statuses_skipped: - LOG.debug("================================") - LOG.debug("Images skipped due build options") - LOG.debug("================================") + LOG.info("===================================") + LOG.info("Images skipped due to build options") + LOG.info("===================================") for name in sorted(self.image_statuses_skipped.keys()): - LOG.debug(name) + LOG.info(name) results['skipped'].append({ 'name': name, }) + if self.image_statuses_unbuildable: + LOG.info("=========================================") + LOG.info("Images not buildable due to build options") + LOG.info("=========================================") + for name in sorted(self.image_statuses_unbuildable.keys()): + LOG.info(name) + results['unbuildable'].append({ + 'name': name, + }) + return results def get_image_statuses(self): if any([self.image_statuses_bad, self.image_statuses_good, self.image_statuses_unmatched, - self.image_statuses_skipped]): + self.image_statuses_skipped, + self.image_statuses_unbuildable]): return (self.image_statuses_bad, self.image_statuses_good, self.image_statuses_unmatched, - self.image_statuses_skipped) + self.image_statuses_skipped, + self.image_statuses_unbuildable) for image in self.images: if image.status == STATUS_BUILT: self.image_statuses_good[image.name] = image.status @@ -1124,12 +1175,15 @@ class KollaWorker(object): self.image_statuses_unmatched[image.name] = image.status elif image.status == STATUS_SKIPPED: self.image_statuses_skipped[image.name] = image.status + elif image.status == STATUS_UNBUILDABLE: + self.image_statuses_unbuildable[image.name] = image.status else: self.image_statuses_bad[image.name] = image.status return (self.image_statuses_bad, self.image_statuses_good, self.image_statuses_unmatched, - self.image_statuses_skipped) + self.image_statuses_skipped, + self.image_statuses_unbuildable) def build_image_list(self): def process_source_installation(image, section): @@ -1285,7 +1339,8 @@ class KollaWorker(object): queue = six.moves.queue.Queue() for image in self.images: - if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED): + if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED, + STATUS_UNBUILDABLE): # Don't bother queuing up build tasks for things that # were not matched in the first place... (not worth the # effort to run them, if they won't be used anyway). diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index ec4ab8f54..72a35029f 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -541,14 +541,14 @@ class MainTest(base.TestCase): @mock.patch.object(build, 'run_build') def test_images_built(self, mock_run_build): - image_statuses = ({}, {'img': 'built'}, {}, {}) + image_statuses = ({}, {'img': 'built'}, {}, {}, {}) mock_run_build.return_value = image_statuses result = build_cmd.main() self.assertEqual(0, result) @mock.patch.object(build, 'run_build') def test_images_unmatched(self, mock_run_build): - image_statuses = ({}, {}, {'img': 'unmatched'}, {}) + image_statuses = ({}, {}, {'img': 'unmatched'}, {}, {}) mock_run_build.return_value = image_statuses result = build_cmd.main() self.assertEqual(0, result) @@ -561,7 +561,7 @@ class MainTest(base.TestCase): @mock.patch.object(build, 'run_build') def test_bad_images(self, mock_run_build): - image_statuses = ({'img': 'error'}, {}, {}, {}) + image_statuses = ({'img': 'error'}, {}, {}, {}, {}) mock_run_build.return_value = image_statuses result = build_cmd.main() self.assertEqual(1, result) @@ -574,7 +574,14 @@ class MainTest(base.TestCase): @mock.patch.object(build, 'run_build') def test_skipped_images(self, mock_run_build): - image_statuses = ({}, {}, {}, {'img': 'skipped'}) + image_statuses = ({}, {}, {}, {'img': 'skipped'}, {}) + mock_run_build.return_value = image_statuses + result = build_cmd.main() + self.assertEqual(0, result) + + @mock.patch.object(build, 'run_build') + def test_unbuildable_images(self, mock_run_build): + image_statuses = ({}, {}, {}, {}, {'img': 'unbuildable'}) mock_run_build.return_value = image_statuses result = build_cmd.main() self.assertEqual(0, result) diff --git a/tests/test_build.py b/tests/test_build.py index d32188b55..d95208968 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -48,11 +48,11 @@ class BuildTest(object): with patch.object(sys, 'argv', self.build_args): LOG.info("Running with args %s", self.build_args) (bad_results, good_results, unmatched_results, - skipped_results) = build.run_build() + skipped_results, unbuildable_results) = build.run_build() failures = 0 for image, result in bad_results.items(): - if result is not 'error': + if result != 'error': continue failures = failures + 1 LOG.critical(">>> Expected image '%s' to succeed!", image) |