Merge "Change how build process treats unbuildable images"

3 files changed, 103 insertions, 41 deletions

diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py
index e912a6d12..3ccc4475a 100755
--- a/kolla/cmd/build.py
+++ b/kolla/cmd/build.py
@@ -33,7 +33,7 @@ def main():
         statuses = build.run_build()
         if statuses:
             (bad_results, good_results, unmatched_results,
-             skipped_results) = statuses
+             skipped_results, unbuildable_results) = statuses
             if bad_results:
                 return 1
         return 0
diff --git a/kolla/image/build.py b/kolla/image/build.py
index d5c51d73b..7b8dcd9af 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -71,16 +71,17 @@ STATUS_UNMATCHED = 'unmatched'
 STATUS_MATCHED = 'matched'
 STATUS_UNPROCESSED = 'unprocessed'
 STATUS_SKIPPED = 'skipped'
+STATUS_UNBUILDABLE = 'unbuildable'
 
 # All error status constants.
 STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
                  STATUS_ERROR, STATUS_PARENT_ERROR)
 
-# The dictionary of skipped images supports keys in the format:
+# The dictionary of unbuildable images supports keys in the format:
 # '<distro>+<installation_type>+<arch>' where each component is optional
 # and can be omitted along with the + separator which means that component
 # is irrelevant. Otherwise all must match for skip to happen.
-SKIPPED_IMAGES = {
+UNBUILDABLE_IMAGES = {
     'aarch64': {
         "cyborg-base",       # no binary package
         "kibana",            # no binary package
@@ -505,7 +506,7 @@ class BuildTask(DockerTask):
 
         self.logger.debug('Processing')
 
-        if image.status == STATUS_SKIPPED:
+        if image.status in [STATUS_SKIPPED, STATUS_UNBUILDABLE]:
             self.logger.info('Skipping %s' % image.name)
             return
 
@@ -741,6 +742,7 @@ class KollaWorker(object):
         self.image_statuses_good = dict()
         self.image_statuses_unmatched = dict()
         self.image_statuses_skipped = dict()
+        self.image_statuses_unbuildable = dict()
         self.maintainer = conf.maintainer
         self.distro_python_version = conf.distro_python_version
 
@@ -1000,44 +1002,80 @@ class KollaWorker(object):
                 else:
                     filter_ += self.conf.profiles[profile]
 
+        # mark unbuildable images and their children
+        tag_element = r'(%s|%s|%s)' % (self.base,
+                                       self.install_type,
+                                       self.base_arch)
+        tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
+        unbuildable_images = set()
+        for set_tag in UNBUILDABLE_IMAGES:
+            if tag_re.match(set_tag):
+                unbuildable_images.update(UNBUILDABLE_IMAGES[set_tag])
+
+        if unbuildable_images:
+            for image in self.images:
+                if image.name in unbuildable_images:
+                    image.status = STATUS_UNBUILDABLE
+                else:
+                    # let's check ancestors
+                    # if any of them is unbuildable then we mark it
+                    # and then mark image
+                    build_image = True
+                    ancestor_image = image
+                    while (ancestor_image.parent is not None):
+                        ancestor_image = ancestor_image.parent
+                        if ancestor_image.name in unbuildable_images or \
+                           ancestor_image.status == STATUS_UNBUILDABLE:
+                            build_image = False
+                            ancestor_image.status = STATUS_UNBUILDABLE
+                            break
+                    if not build_image:
+                        image.status = STATUS_UNBUILDABLE
+
+        # When we want to build a subset of images then filter_ part kicks in.
+        # Otherwise we just mark everything buildable as matched for build.
+
         if filter_:
             patterns = re.compile(r"|".join(filter_).join('()'))
             for image in self.images:
-                if image.status in (STATUS_MATCHED, STATUS_SKIPPED):
+                # as we now list not buildable/skipped images we need to
+                # process them otherwise list will contain also not requested
+                # entries
+                if image.status == STATUS_MATCHED:
                     continue
                 if re.search(patterns, image.name):
-                    image.status = STATUS_MATCHED
-                    while (image.parent is not None and
-                           image.parent.status not in (STATUS_MATCHED,
-                                                       STATUS_SKIPPED)):
-                        image = image.parent
+                    if image.status not in [STATUS_SKIPPED,
+                                            STATUS_UNBUILDABLE]:
+                        image.status = STATUS_MATCHED
+
+                    # skip image if --skip-existing was given and image
+                    # was already built
+                    if (self.conf.skip_existing and image.in_docker_cache()):
+                        image.status = STATUS_SKIPPED
+
+                    # handle image ancestors
+                    ancestor_image = image
+                    while (ancestor_image.parent is not None and
+                           ancestor_image.parent.status not in
+                           (STATUS_MATCHED, STATUS_SKIPPED)):
+                        ancestor_image = ancestor_image.parent
                         if self.conf.skip_parents:
-                            image.status = STATUS_SKIPPED
+                            ancestor_image.status = STATUS_SKIPPED
                         elif (self.conf.skip_existing and
-                              image.in_docker_cache()):
-                            image.status = STATUS_SKIPPED
+                              ancestor_image.in_docker_cache()):
+                            ancestor_image.status = STATUS_SKIPPED
                         else:
-                            image.status = STATUS_MATCHED
+                            if ancestor_image.status != STATUS_UNBUILDABLE:
+                                ancestor_image.status = STATUS_MATCHED
                         LOG.debug('Image %s matched regex', image.name)
                 else:
+                    # we do not care if it is skipped or not as we did not
+                    # request it
                     image.status = STATUS_UNMATCHED
         else:
             for image in self.images:
-                image.status = STATUS_MATCHED
-
-        # Unmatch (skip) unsupported images
-        tag_element = r'(%s|%s|%s)' % (self.base,
-                                       self.install_type,
-                                       self.base_arch)
-        tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
-        skipped_images = set()
-        for set_tag in SKIPPED_IMAGES:
-            if tag_re.match(set_tag):
-                skipped_images.update(SKIPPED_IMAGES[set_tag])
-        if skipped_images:
-            for image in self.images:
-                if image.name in skipped_images:
-                    image.status = STATUS_UNMATCHED
+                if image.status != STATUS_UNBUILDABLE:
+                    image.status = STATUS_MATCHED
 
     def summary(self):
         """Walk the dictionary of images statuses and print results."""
@@ -1053,6 +1091,7 @@ class KollaWorker(object):
             'failed': [],
             'not_matched': [],
             'skipped': [],
+            'unbuildable': [],
         }
 
         if self.image_statuses_good:
@@ -1097,26 +1136,38 @@ class KollaWorker(object):
                 })
 
         if self.image_statuses_skipped:
-            LOG.debug("================================")
-            LOG.debug("Images skipped due build options")
-            LOG.debug("================================")
+            LOG.info("===================================")
+            LOG.info("Images skipped due to build options")
+            LOG.info("===================================")
             for name in sorted(self.image_statuses_skipped.keys()):
-                LOG.debug(name)
+                LOG.info(name)
                 results['skipped'].append({
                     'name': name,
                 })
 
+        if self.image_statuses_unbuildable:
+            LOG.info("=========================================")
+            LOG.info("Images not buildable due to build options")
+            LOG.info("=========================================")
+            for name in sorted(self.image_statuses_unbuildable.keys()):
+                LOG.info(name)
+                results['unbuildable'].append({
+                    'name': name,
+                })
+
         return results
 
     def get_image_statuses(self):
         if any([self.image_statuses_bad,
                 self.image_statuses_good,
                 self.image_statuses_unmatched,
-                self.image_statuses_skipped]):
+                self.image_statuses_skipped,
+                self.image_statuses_unbuildable]):
             return (self.image_statuses_bad,
                     self.image_statuses_good,
                     self.image_statuses_unmatched,
-                    self.image_statuses_skipped)
+                    self.image_statuses_skipped,
+                    self.image_statuses_unbuildable)
         for image in self.images:
             if image.status == STATUS_BUILT:
                 self.image_statuses_good[image.name] = image.status
@@ -1124,12 +1175,15 @@ class KollaWorker(object):
                 self.image_statuses_unmatched[image.name] = image.status
             elif image.status == STATUS_SKIPPED:
                 self.image_statuses_skipped[image.name] = image.status
+            elif image.status == STATUS_UNBUILDABLE:
+                self.image_statuses_unbuildable[image.name] = image.status
             else:
                 self.image_statuses_bad[image.name] = image.status
         return (self.image_statuses_bad,
                 self.image_statuses_good,
                 self.image_statuses_unmatched,
-                self.image_statuses_skipped)
+                self.image_statuses_skipped,
+                self.image_statuses_unbuildable)
 
     def build_image_list(self):
         def process_source_installation(image, section):
@@ -1285,7 +1339,8 @@ class KollaWorker(object):
         queue = six.moves.queue.Queue()
 
         for image in self.images:
-            if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED):
+            if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED,
+                                STATUS_UNBUILDABLE):
                 # Don't bother queuing up build tasks for things that
                 # were not matched in the first place... (not worth the
                 # effort to run them, if they won't be used anyway).
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index ec4ab8f54..72a35029f 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -541,14 +541,14 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_images_built(self, mock_run_build):
-        image_statuses = ({}, {'img': 'built'}, {}, {})
+        image_statuses = ({}, {'img': 'built'}, {}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)
 
     @mock.patch.object(build, 'run_build')
     def test_images_unmatched(self, mock_run_build):
-        image_statuses = ({}, {}, {'img': 'unmatched'}, {})
+        image_statuses = ({}, {}, {'img': 'unmatched'}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)
@@ -561,7 +561,7 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_bad_images(self, mock_run_build):
-        image_statuses = ({'img': 'error'}, {}, {}, {})
+        image_statuses = ({'img': 'error'}, {}, {}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(1, result)
@@ -574,7 +574,14 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_skipped_images(self, mock_run_build):
-        image_statuses = ({}, {}, {}, {'img': 'skipped'})
+        image_statuses = ({}, {}, {}, {'img': 'skipped'}, {})
+        mock_run_build.return_value = image_statuses
+        result = build_cmd.main()
+        self.assertEqual(0, result)
+
+    @mock.patch.object(build, 'run_build')
+    def test_unbuildable_images(self, mock_run_build):
+        image_statuses = ({}, {}, {}, {}, {'img': 'unbuildable'})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)