Merge "Add a way to add other container engines"

7 files changed, 150 insertions, 67 deletions

diff --git a/kolla/common/config.py b/kolla/common/config.py
index 87ecf9d50..b9e801a73 100644
--- a/kolla/common/config.py
+++ b/kolla/common/config.py
@@ -156,7 +156,8 @@ _CLI_OPTS = [
     cfg.BoolOpt('skip-parents', default=False,
                 help='Do not rebuild parents of matched images'),
     cfg.BoolOpt('skip-existing', default=False,
-                help='Do not rebuild images present in the docker cache'),
+                help='Do not rebuild images present in the container engine '
+                'cache'),
     cfg.DictOpt('build-args',
                 help='Set docker build time variables'),
     cfg.BoolOpt('keep', default=False,
@@ -172,7 +173,7 @@ _CLI_OPTS = [
     cfg.StrOpt('network_mode', default='host',
                help='The network mode for Docker build. Example: host'),
     cfg.BoolOpt('cache', default=True,
-                help='Use the Docker cache when building'),
+                help='Use the container engine cache when building'),
     cfg.MultiOpt('profile', types.String(), short='p',
                  help=('Build a pre-defined set of images, see [profiles]'
                        ' section in config. The default profiles are:'
@@ -190,14 +191,14 @@ _CLI_OPTS = [
                  help=('Build only images matching regex and its'
                        ' dependencies')),
     cfg.StrOpt('registry',
-               help=('The docker registry host. The default registry host'
-                     ' is Docker Hub')),
+               help=('The container image registry host. The default registry'
+                     ' host is Docker Hub')),
     cfg.StrOpt('save-dependency',
                help=('Path to the file to store the docker image'
                      ' dependency in Graphviz dot format')),
     cfg.StrOpt('format', short='f', default='json',
                choices=['json', 'none'],
-               help='Format to write the final results in'),
+               help='Format to write the final results in.'),
     cfg.StrOpt('tarballs-base', default=TARBALLS_BASE,
                help='Base url to OpenStack tarballs'),
     cfg.IntOpt('threads', short='T', default=8, min=1,
@@ -205,7 +206,7 @@ _CLI_OPTS = [
                      ' (Note: setting to one will allow real time'
                      ' logging)')),
     cfg.StrOpt('tag', default=version.cached_version_string(),
-               help='The Docker tag'),
+               help='The container image tag'),
     cfg.BoolOpt('template-only', default=False,
                 help="Don't build images. Generate Dockerfile only"),
     cfg.IntOpt('timeout', default=120,
@@ -246,6 +247,8 @@ _CLI_OPTS = [
                help='Prefix prepended to image names'),
     cfg.StrOpt('repos-yaml', default='',
                help='Path to alternative repos.yaml file'),
+    cfg.StrOpt('engine', default='docker', choices=['docker'],
+               help='Container engine to build images on.')
 ]
 
 _BASE_OPTS = [
diff --git a/kolla/engine_adapter/__init__.py b/kolla/engine_adapter/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/kolla/engine_adapter/__init__.py
diff --git a/kolla/engine_adapter/engine.py b/kolla/engine_adapter/engine.py
new file mode 100644
index 000000000..184bb3856
--- /dev/null
+++ b/kolla/engine_adapter/engine.py
@@ -0,0 +1,57 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distutils.version import StrictVersion
+from enum import Enum
+from kolla.image.utils import LOG
+
+try:
+    import docker
+except (ImportError):
+    LOG.debug("Docker python library was not found")
+
+
+class Engine(Enum):
+
+    DOCKER = "docker"
+
+
+class UnsupportedEngineError(ValueError):
+
+    def __init__(self, engine_name):
+        super().__init__()
+        self.engine_name = engine_name
+
+    def __str__(self):
+        return f'Unsupported engine name given: "{self.engine_name}"'
+
+
+def getEngineException(conf):
+    if conf.engine == Engine.DOCKER.value:
+        return (docker.errors.DockerException)
+    else:
+        raise UnsupportedEngineError(conf.engine)
+
+
+def getEngineClient(conf):
+    if conf.engine == Engine.DOCKER.value:
+        kwargs_env = docker.utils.kwargs_from_env()
+        return docker.APIClient(version='auto', **kwargs_env)
+    else:
+        raise UnsupportedEngineError(conf.engine)
+
+
+def getEngineVersion(conf):
+    if conf.engine == Engine.DOCKER.value:
+        return StrictVersion(docker.version)
+    else:
+        raise UnsupportedEngineError(conf.engine)
diff --git a/kolla/image/build.py b/kolla/image/build.py
index 7306e5fdc..fdf1635c8 100644
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -21,6 +21,7 @@ import time
 
 from kolla.common import config as common_config
 from kolla.common import utils
+from kolla.engine_adapter import engine
 from kolla.image.kolla_worker import KollaWorker
 from kolla.image.utils import LOG
 from kolla.image.utils import Status
@@ -104,10 +105,28 @@ def run_build():
     if conf.debug:
         LOG.setLevel(logging.DEBUG)
 
-    if conf.squash:
-        squash_version = utils.get_docker_squash_version()
-        LOG.info('Image squash is enabled and "docker-squash" version is %s',
-                 squash_version)
+    if conf.engine not in (engine.Engine.DOCKER.value,):
+        LOG.error(f'Unsupported engine name "{conf.engine}", exiting.')
+        sys.exit(1)
+    LOG.info(f'Using engine: {conf.engine}')
+
+    if conf.engine == engine.Engine.DOCKER.value:
+        try:
+            import docker
+            docker.version
+        except ImportError:
+            LOG.error("Error, you have set Docker as container engine, "
+                      "but the Python library is not found."
+                      "Try running 'pip install docker'")
+            sys.exit(1)
+        except AttributeError:
+            LOG.error("Error, Docker Python library is too old, "
+                      "Try running 'pip install docker --upgrade'")
+
+        if conf.squash:
+            squash_version = utils.get_docker_squash_version()
+            LOG.info('Image squash is enabled and "docker-squash" version '
+                     'is %s', squash_version)
 
     kolla = KollaWorker(conf)
     kolla.setup_working_dir()
@@ -133,7 +152,7 @@ def run_build():
 
     if conf.save_dependency:
         kolla.save_dependency(conf.save_dependency)
-        LOG.info('Docker images dependency are saved in %s',
+        LOG.info('Container images dependency are saved in %s',
                  conf.save_dependency)
         return
     if conf.list_images:
diff --git a/kolla/image/kolla_worker.py b/kolla/image/kolla_worker.py
index b1e7ac943..1a665ddd4 100644
--- a/kolla/image/kolla_worker.py
+++ b/kolla/image/kolla_worker.py
@@ -11,7 +11,6 @@
 # limitations under the License.
 
 import datetime
-import docker
 import json
 import os
 import queue
@@ -24,6 +23,7 @@ import time
 import jinja2
 from kolla.common import config as common_config
 from kolla.common import utils
+from kolla.engine_adapter import engine
 from kolla import exception
 from kolla.image.tasks import BuildTask
 from kolla.image.unbuildable import UNBUILDABLE_IMAGES
@@ -42,7 +42,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(
 class Image(object):
     def __init__(self, name, canonical_name, path, parent_name='',
                  status=Status.UNPROCESSED, parent=None,
-                 source=None, logger=None, docker_client=None):
+                 source=None, logger=None, engine_client=None):
         self.name = name
         self.canonical_name = canonical_name
         self.path = path
@@ -56,7 +56,7 @@ class Image(object):
         self.children = []
         self.plugins = []
         self.additions = []
-        self.dc = docker_client
+        self.engine_client = engine_client
 
     def copy(self):
         c = Image(self.name, self.canonical_name, self.path,
@@ -72,8 +72,9 @@ class Image(object):
             c.additions = list(self.additions)
         return c
 
-    def in_docker_cache(self):
-        return len(self.dc.images(name=self.canonical_name, quiet=True)) == 1
+    def in_engine_cache(self):
+        return len(self.engine_client.images(name=self.canonical_name,
+                                             quiet=True)) == 1
 
     def __repr__(self):
         return ("Image(%s, %s, %s, parent_name=%s,"
@@ -148,16 +149,16 @@ class KollaWorker(object):
         self.maintainer = conf.maintainer
         self.distro_python_version = conf.distro_python_version
 
-        docker_kwargs = docker.utils.kwargs_from_env()
         try:
-            self.dc = docker.APIClient(version='auto', **docker_kwargs)
-        except docker.errors.DockerException as e:
-            self.dc = None
+            self.engine_client = engine.getEngineClient(self.conf)
+        except engine.getEngineException(self.conf) as e:
+            self.engine_client = None
             if not (conf.template_only or
                     conf.save_dependency or
                     conf.list_images or
                     conf.list_dependencies):
-                LOG.error("Unable to connect to Docker, exiting")
+                LOG.error("Unable to connect to container engine daemon, "
+                          "exiting")
                 LOG.info("Exception caught: {0}".format(e))
                 sys.exit(1)
 
@@ -179,18 +180,18 @@ class KollaWorker(object):
             #                 this is the correct path
             # TODO(SamYaple): Improve this to make this safer
             if os.path.exists(os.path.join(image_path, 'base')):
-                LOG.info('Found the docker image folder at %s', image_path)
+                LOG.info('Found the container image folder at %s', image_path)
                 return image_path
         else:
             raise exception.KollaDirNotFoundException('Image dir can not '
                                                       'be found')
 
     def build_rpm_setup(self, rpm_setup_config):
-        """Generates a list of docker commands based on provided configuration.
+        """Generates a list of engine commands based on provided configuration
 
         :param rpm_setup_config: A list of .rpm or .repo paths or URLs
                                  (can be empty)
-        :return: A list of docker commands
+        :return: A list of engine commands
         """
         rpm_setup = list()
 
@@ -470,7 +471,7 @@ class KollaWorker(object):
             if image.status != Status.MATCHED:
                 continue
             # Skip image if --skip-existing was given and image exists.
-            if (self.conf.skip_existing and image.in_docker_cache()):
+            if (self.conf.skip_existing and image.in_engine_cache()):
                 LOG.debug('Skipping existing image %s', image.name)
                 image.status = Status.SKIPPED
             # Skip image if --skip-parents was given and image has children.
@@ -638,7 +639,7 @@ class KollaWorker(object):
             image = Image(image_name, canonical_name, path,
                           parent_name=parent_name,
                           logger=utils.make_a_logger(self.conf, image_name),
-                          docker_client=self.dc)
+                          engine_client=self.engine_client)
 
             # NOTE(jeffrey4l): register the opts if the section didn't
             # register in the kolla/common/config.py file
@@ -683,7 +684,7 @@ class KollaWorker(object):
         except ImportError:
             LOG.error('"graphviz" is required for save dependency')
             raise
-        dot = graphviz.Digraph(comment='Docker Images Dependency')
+        dot = graphviz.Digraph(comment='Container Images Dependency')
         dot.body.extend(['rankdir=LR'])
         for image in self.images:
             if image.status not in [Status.MATCHED]:
diff --git a/kolla/image/tasks.py b/kolla/image/tasks.py
index bfc01dec5..5038c1d52 100644
--- a/kolla/image/tasks.py
+++ b/kolla/image/tasks.py
@@ -11,7 +11,6 @@
 # limitations under the License.
 
 import datetime
-import docker
 import errno
 import os
 import shutil
@@ -23,6 +22,7 @@ from requests import exceptions as requests_exc
 
 from kolla.common import task  # noqa
 from kolla.common import utils  # noqa
+from kolla.engine_adapter import engine
 from kolla.image.utils import Status
 from kolla.image.utils import STATUS_ERRORS
 
@@ -31,21 +31,18 @@ class ArchivingError(Exception):
     pass
 
 
-class DockerTask(task.Task):
-
-    docker_kwargs = docker.utils.kwargs_from_env()
-
-    def __init__(self):
-        super(DockerTask, self).__init__()
-        self._dc = None
+class EngineTask(task.Task):
+    def __init__(self, conf):
+        super(EngineTask, self).__init__()
+        self._ec = None
+        self.conf = conf
 
     @property
-    def dc(self):
-        if self._dc is not None:
-            return self._dc
-        docker_kwargs = self.docker_kwargs.copy()
-        self._dc = docker.APIClient(version='auto', **docker_kwargs)
-        return self._dc
+    def engine_client(self):
+        if self._ec is not None:
+            return self._ec
+        self._ec = engine.getEngineClient(self.conf)
+        return self._ec
 
 
 class PushIntoQueueTask(task.Task):
@@ -70,11 +67,11 @@ class PushError(Exception):
     pass
 
 
-class PushTask(DockerTask):
-    """Task that pushes an image to a docker repository."""
+class PushTask(EngineTask):
+    """Task that pushes an image to a container image repository."""
 
     def __init__(self, conf, image):
-        super(PushTask, self).__init__()
+        super(PushTask, self).__init__(conf)
         self.conf = conf
         self.image = image
         self.logger = image.logger
@@ -89,9 +86,10 @@ class PushTask(DockerTask):
         try:
             self.push_image(image)
         except requests_exc.ConnectionError:
-            self.logger.exception('Make sure Docker is running and that you'
-                                  ' have the correct privileges to run Docker'
-                                  ' (root)')
+            self.logger.exception('Make sure container engine daemon is '
+                                  'running and that you have the correct '
+                                  'privileges to run the '
+                                  'container engine (root)')
             image.status = Status.CONNECTION_ERROR
         except PushError as exception:
             self.logger.error(exception)
@@ -110,7 +108,8 @@ class PushTask(DockerTask):
     def push_image(self, image):
         kwargs = dict(stream=True, decode=True)
 
-        for response in self.dc.push(image.canonical_name, **kwargs):
+        for response in self.engine_client.push(
+                image.canonical_name, **kwargs):
             if 'stream' in response:
                 self.logger.info(response['stream'])
             elif 'errorDetail' in response:
@@ -120,11 +119,11 @@ class PushTask(DockerTask):
         image.status = Status.BUILT
 
 
-class BuildTask(DockerTask):
+class BuildTask(EngineTask):
     """Task that builds out an image."""
 
     def __init__(self, conf, image, push_queue):
-        super(BuildTask, self).__init__()
+        super(BuildTask, self).__init__(conf)
         self.conf = conf
         self.image = image
         self.push_queue = push_queue
@@ -145,8 +144,9 @@ class BuildTask(DockerTask):
         followups = []
         if self.conf.push and self.success:
             followups.extend([
-                # If we are supposed to push the image into a docker
-                # repository, then make sure we do that...
+                # If we are supposed to push the image into a
+                # container image repository,
+                # then make sure we do that...
                 PushIntoQueueTask(
                     PushTask(self.conf, self.image),
                     self.push_queue),
@@ -358,15 +358,16 @@ class BuildTask(DockerTask):
 
         buildargs = self.update_buildargs()
         try:
-            for stream in self.dc.build(path=image.path,
-                                        tag=image.canonical_name,
-                                        nocache=not self.conf.cache,
-                                        rm=True,
-                                        decode=True,
-                                        network_mode=self.conf.network_mode,
-                                        pull=pull,
-                                        forcerm=self.forcerm,
-                                        buildargs=buildargs):
+            for stream in \
+                self.engine_client.build(path=image.path,
+                                         tag=image.canonical_name,
+                                         nocache=not self.conf.cache,
+                                         rm=True,
+                                         decode=True,
+                                         network_mode=self.conf.network_mode,
+                                         pull=pull,
+                                         forcerm=self.forcerm,
+                                         buildargs=buildargs):
                 if 'stream' in stream:
                     for line in stream['stream'].split('\n'):
                         if line:
@@ -379,11 +380,13 @@ class BuildTask(DockerTask):
                             self.logger.error('%s', line)
                     return
 
-            if image.status != Status.ERROR and self.conf.squash:
+            if image.status != Status.ERROR and self.conf.squash and \
+               self.conf.engine == engine.Engine.DOCKER.value:
                 self.squash()
-        except docker.errors.DockerException:
+        except engine.getEngineException(self.conf):
             image.status = Status.ERROR
-            self.logger.exception('Unknown docker error when building')
+            self.logger.exception('Unknown container engine '
+                                  'error when building')
         except Exception:
             image.status = Status.ERROR
             self.logger.exception('Unknown error when building')
@@ -395,9 +398,9 @@ class BuildTask(DockerTask):
 
     def squash(self):
         image_tag = self.image.canonical_name
-        image_id = self.dc.inspect_image(image_tag)['Id']
+        image_id = self.engine_client.inspect_image(image_tag)['Id']
 
-        parent_history = self.dc.history(self.image.parent_name)
+        parent_history = self.engine_client.history(self.image.parent_name)
         parent_last_layer = parent_history[0]['Id']
         self.logger.info('Parent lastest layer is: %s' % parent_last_layer)
 
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index c4c560c2f..a4784ba86 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -493,7 +493,7 @@ class KollaWorkerTest(base.TestCase):
         self.assertEqual(utils.Status.SKIPPED, kolla.images[2].parent.status)
         self.assertEqual(utils.Status.SKIPPED, kolla.images[1].parent.status)
 
-    @mock.patch.object(Image, 'in_docker_cache')
+    @mock.patch.object(Image, 'in_engine_cache')
     def test_skip_existing(self, mock_in_cache):
         mock_in_cache.side_effect = [True, False]
         self.conf.set_override('skip_existing', True)