diff options
| -rw-r--r-- | kolla/common/config.py | 6 | ||||
| -rw-r--r-- | kolla/engine_adapter/engine.py | 20 | ||||
| -rw-r--r-- | kolla/image/build.py | 14 | ||||
| -rw-r--r-- | kolla/image/tasks.py | 29 | ||||
| -rw-r--r-- | kolla/tests/test_build.py | 59 | ||||
| -rw-r--r-- | releasenotes/notes/add-docker-engine-7402d3fe7349f1f3.yaml | 7 | ||||
| -rw-r--r-- | tox.ini | 11 |
7 files changed, 117 insertions, 29 deletions
diff --git a/kolla/common/config.py b/kolla/common/config.py index 4f3b9da75..0979d2c7c 100644 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -246,8 +246,10 @@ _CLI_OPTS = [ help='Prefix prepended to image names'), cfg.StrOpt('repos-yaml', default='', help='Path to alternative repos.yaml file'), - cfg.StrOpt('engine', default='docker', choices=['docker'], - help='Container engine to build images on.') + cfg.StrOpt('engine', default='docker', choices=['docker', 'podman'], + help='Container engine to build images on.'), + cfg.StrOpt('podman_base_url', default='unix:///run/podman/podman.sock', + help='Path to podman socket.') ] _BASE_OPTS = [ diff --git a/kolla/engine_adapter/engine.py b/kolla/engine_adapter/engine.py index 95966c18a..dc38f4541 100644 --- a/kolla/engine_adapter/engine.py +++ b/kolla/engine_adapter/engine.py @@ -18,10 +18,17 @@ try: except (ImportError): LOG.debug("Docker python library was not found") +try: + import podman +except (ImportError, ModuleNotFoundError): + LOG.debug("Podman python library was not found") + pass + class Engine(Enum): DOCKER = "docker" + PODMAN = "podman" class UnsupportedEngineError(ValueError): @@ -37,6 +44,9 @@ class UnsupportedEngineError(ValueError): def getEngineException(conf): if conf.engine == Engine.DOCKER.value: return (docker.errors.DockerException) + elif conf.engine == Engine.PODMAN.value: + return (podman.errors.exceptions.APIError, + podman.errors.exceptions.PodmanError) else: raise UnsupportedEngineError(conf.engine) @@ -45,5 +55,15 @@ def getEngineClient(conf): if conf.engine == Engine.DOCKER.value: kwargs_env = docker.utils.kwargs_from_env() return docker.DockerClient(version='auto', **kwargs_env) + elif conf.engine == Engine.PODMAN.value: + client = podman.PodmanClient(base_url=conf.podman_base_url) + try: + client.version() + except podman.errors.exceptions.APIError as e: + e.explanation += (". Check if podman service is active and " + "the address to podman.sock is set correctly " + "through --podman_base_url") + raise e + return client else: raise UnsupportedEngineError(conf.engine) diff --git a/kolla/image/build.py b/kolla/image/build.py index 41cc11708..f75ca64c8 100644 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -106,7 +106,8 @@ def run_build(): if conf.debug: LOG.setLevel(logging.DEBUG) - if conf.engine not in (engine.Engine.DOCKER.value,): + if conf.engine not in (engine.Engine.DOCKER.value, + engine.Engine.PODMAN.value): LOG.error(f'Unsupported engine name "{conf.engine}", exiting.') sys.exit(1) LOG.info(f'Using engine: {conf.engine}') @@ -123,12 +124,21 @@ def run_build(): except AttributeError: LOG.error("Error, Docker Python library is too old, " "Try running 'pip install docker --upgrade'") - if conf.squash: squash_version = utils.get_docker_squash_version() LOG.info('Image squash is enabled and "docker-squash" version ' 'is %s', squash_version) + if conf.engine == engine.Engine.PODMAN.value: + try: + import podman + podman.__version__ + except ImportError: + LOG.error("Error, you have set podman as container engine, " + "but library is not found." + "Try running pip install podman") + exit(1) + kolla = KollaWorker(conf) kolla.setup_working_dir() kolla.find_dockerfiles() diff --git a/kolla/image/tasks.py b/kolla/image/tasks.py index 1fbfbd6b5..ada30d664 100644 --- a/kolla/image/tasks.py +++ b/kolla/image/tasks.py @@ -12,11 +12,16 @@ import datetime import errno +import json import os import shutil import tarfile import docker.errors +try: + import podman.errors.exceptions +except (ImportError, ModuleNotFoundError): + pass import git import requests from requests import exceptions as requests_exc @@ -367,6 +372,20 @@ class BuildTask(EngineTask): pull = self.conf.pull if image.parent is None else False buildargs = self.update_buildargs() + + kwargs = {} + if self.conf.engine == engine.Engine.PODMAN.value: + # TODO(kevko): dockerfile path is a workaround, + # should be removed as soon as it will be fixed in podman-py + # https://github.com/containers/podman-py/issues/177 + kwargs["dockerfile"] = image.path + '/Dockerfile' + # Podman squash is different by default + # https://github.com/containers/buildah/issues/1234 + if self.conf.squash: + kwargs["squash"] = False + kwargs["layers"] = False + else: + kwargs["layers"] = True try: for stream in self.engine_client.images.build( path=image.path, @@ -376,7 +395,10 @@ class BuildTask(EngineTask): network_mode=self.conf.network_mode, pull=pull, forcerm=self.forcerm, - buildargs=buildargs)[1]: + buildargs=buildargs, + **kwargs)[1]: + if self.conf.engine == engine.Engine.PODMAN.value: + stream = json.loads(stream) if 'stream' in stream: for line in stream['stream'].split('\n'): if line: @@ -399,6 +421,11 @@ class BuildTask(EngineTask): for line in e.build_log: if 'stream' in line: self.logger.error(line['stream'].strip()) + if isinstance(e, podman.errors.exceptions.BuildError): + for line in e.build_log: + line = json.loads(line) + if 'stream' in line: + self.logger.error(line['stream'].strip()) self.logger.exception('Unknown container engine ' 'error when building') diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 1ffa34f68..8fd0a5d9a 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -50,6 +50,9 @@ FAKE_IMAGE_GRANDCHILD = Image( 'image-grandchild', 'image-grandchild:latest', '/fake/path6', parent_name='image-child', parent=FAKE_IMAGE_CHILD, status=utils.Status.MATCHED) +engine_client = "docker.DockerClient" +if os.getenv("TEST_ENGINE") == "podman": + engine_client = "podman.PodmanClient" class TasksTest(base.TestCase): @@ -63,9 +66,19 @@ class TasksTest(base.TestCase): # NOTE(mandre) we want the local copy of FAKE_IMAGE as the parent self.imageChild.parent = self.image self.imageChild.path = self.useFixture(fixtures.TempDir()).path + if "podman" in engine_client: + self.conf.set_override('engine', 'podman') + self.build_kwargs["dockerfile"] = self.image.path + '/Dockerfile' + # NOTE(kevko): Squash implementation is different in podman + # + # - Check kolla/image/tasks.py for podman + # - Check https://github.com/containers/buildah/issues/1234 + self.build_kwargs["squash"] = False + else: + self.build_kwargs = {} @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_push_image(self, mock_client): self.engine_client = mock_client pusher = tasks.PushTask(self.conf, self.image) @@ -75,7 +88,7 @@ class TasksTest(base.TestCase): self.assertTrue(pusher.success) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_push_image_failure(self, mock_client): """failure on connecting Docker API""" self.engine_client = mock_client @@ -88,7 +101,7 @@ class TasksTest(base.TestCase): self.assertEqual(utils.Status.PUSH_ERROR, self.image.status) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_push_image_failure_retry(self, mock_client): """failure on connecting Docker API, success on retry""" self.engine_client = mock_client @@ -108,7 +121,7 @@ class TasksTest(base.TestCase): self.assertEqual(utils.Status.BUILT, self.image.status) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_push_image_failure_error(self, mock_client): """Docker connected, failure to push""" self.engine_client = mock_client @@ -122,7 +135,7 @@ class TasksTest(base.TestCase): self.assertEqual(utils.Status.PUSH_ERROR, self.image.status) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_push_image_failure_error_retry(self, mock_client): """Docker connected, failure to push, success on retry""" self.engine_client = mock_client @@ -145,7 +158,7 @@ class TasksTest(base.TestCase): self.assertEqual(utils.Status.BUILT, self.image.status) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_build_image(self, mock_client): self.engine_client = mock_client push_queue = mock.Mock() @@ -155,12 +168,12 @@ class TasksTest(base.TestCase): mock_client().images.build.assert_called_once_with( path=self.image.path, tag=self.image.canonical_name, network_mode='host', nocache=False, rm=True, pull=True, - forcerm=True, buildargs=None) + forcerm=True, buildargs=None, **self.build_kwargs) self.assertTrue(builder.success) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_build_image_with_network_mode(self, mock_client): self.engine_client = mock_client push_queue = mock.Mock() @@ -172,12 +185,12 @@ class TasksTest(base.TestCase): mock_client().images.build.assert_called_once_with( path=self.image.path, tag=self.image.canonical_name, network_mode='bridge', nocache=False, rm=True, pull=True, - forcerm=True, buildargs=None) + forcerm=True, buildargs=None, **self.build_kwargs) self.assertTrue(builder.success) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_build_image_with_build_arg(self, mock_client): self.engine_client = mock_client build_args = { @@ -192,13 +205,13 @@ class TasksTest(base.TestCase): mock_client().images.build.assert_called_once_with( path=self.image.path, tag=self.image.canonical_name, network_mode='host', nocache=False, rm=True, pull=True, - forcerm=True, buildargs=build_args) + forcerm=True, buildargs=build_args, **self.build_kwargs) self.assertTrue(builder.success) @mock.patch.dict(os.environ, {'http_proxy': 'http://FROM_ENV:8080'}, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_build_arg_from_env(self, mock_client): push_queue = mock.Mock() self.engine_client = mock_client @@ -211,13 +224,13 @@ class TasksTest(base.TestCase): mock_client().images.build.assert_called_once_with( path=self.image.path, tag=self.image.canonical_name, network_mode='host', nocache=False, rm=True, pull=True, - forcerm=True, buildargs=build_args) + forcerm=True, buildargs=build_args, **self.build_kwargs) self.assertTrue(builder.success) @mock.patch.dict(os.environ, {'http_proxy': 'http://FROM_ENV:8080'}, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_build_arg_precedence(self, mock_client): self.engine_client = mock_client build_args = { @@ -232,11 +245,11 @@ class TasksTest(base.TestCase): mock_client().images.build.assert_called_once_with( path=self.image.path, tag=self.image.canonical_name, network_mode='host', nocache=False, rm=True, pull=True, - forcerm=True, buildargs=build_args) + forcerm=True, buildargs=build_args, **self.build_kwargs) self.assertTrue(builder.success) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) @mock.patch('requests.get') def test_requests_get_timeout(self, mock_get, mock_client): self.engine_client = mock_client @@ -261,7 +274,7 @@ class TasksTest(base.TestCase): @mock.patch('os.utime') @mock.patch('shutil.copyfile') @mock.patch('shutil.rmtree') - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) @mock.patch('requests.get') def test_process_source(self, mock_get, mock_client, mock_rmtree, mock_copyfile, mock_utime): @@ -293,7 +306,7 @@ class TasksTest(base.TestCase): self.assertIsNotNone(get_result) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_local_directory(self, mock_client): tmpdir = tempfile.mkdtemp() file_name = 'test.txt' @@ -326,7 +339,7 @@ class TasksTest(base.TestCase): self.assertTrue(builder.success) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_malicious_tar(self, mock_client): tmpdir = tempfile.mkdtemp() file_name = 'test.txt' @@ -366,7 +379,7 @@ class TasksTest(base.TestCase): os.rmdir(tmpdir) @mock.patch.dict(os.environ, clear=True) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_malicious_tar_gz(self, mock_client): tmpdir = tempfile.mkdtemp() file_name = 'test.txt' @@ -428,7 +441,7 @@ class TasksTest(base.TestCase): self.assertFalse(builder.success) self.assertIsNone(get_result) - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_followups_docker_image(self, mock_client): self.imageChild.source = { 'source': 'http://fake/source', @@ -463,7 +476,7 @@ class KollaWorkerTest(base.TestCase): self.images = [image, image_child, image_unmatched, image_error, image_built] - patcher = mock.patch('docker.DockerClient') + patcher = mock.patch(engine_client) self.addCleanup(patcher.stop) self.mock_client = patcher.start() @@ -767,7 +780,7 @@ class MainTest(base.TestCase): self.assertEqual(1, result) @mock.patch('sys.argv') - @mock.patch('docker.DockerClient') + @mock.patch(engine_client) def test_run_build(self, mock_client, mock_sys): result = build.run_build() self.assertTrue(result) diff --git a/releasenotes/notes/add-docker-engine-7402d3fe7349f1f3.yaml b/releasenotes/notes/add-docker-engine-7402d3fe7349f1f3.yaml new file mode 100644 index 000000000..51a383fe9 --- /dev/null +++ b/releasenotes/notes/add-docker-engine-7402d3fe7349f1f3.yaml @@ -0,0 +1,7 @@ +--- +features: + - A new engine command-line option has been added to + the client, to allow the user to specify a container + engine to be used for image build, instead of the default + docker container engine. Kolla now supports both docker + and podman container engines. @@ -1,7 +1,7 @@ [tox] minversion = 3.18 skipsdist = True -envlist = pep8,py38 +envlist = pep8,py310-{docker,podman} ignore_basepython_conflict = True [testenv] @@ -15,12 +15,21 @@ deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/up passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY OS_STDOUT_CAPTURE,OS_STDERR_CAPTURE,OS_LOG_CAPTURE,OS_TEST_TIMEOUT PYTHON,OS_TEST_PATH,LISTOPT,IDOPTION + commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete find . -type d -name "__pycache__" -delete stestr run {posargs} stestr slowest +[testenv:docker] +setenv = + TEST_ENGINE=docker + +[testenv:podman] +setenv = + TEST_ENGINE=podman + [testenv:debug] commands = oslo_debug_helper -t kolla/tests {posargs} |