Merge "Add systemd container control"

9 files changed, 589 insertions, 30 deletions

diff --git a/ansible/gather-facts.yml b/ansible/gather-facts.yml
index f5247a6ab..b50389660 100644
--- a/ansible/gather-facts.yml
+++ b/ansible/gather-facts.yml
@@ -15,6 +15,13 @@
       when:
         - not ansible_facts
 
+    - name: Gather package facts
+      package_facts:
+      when:
+        - "'packages' not in ansible_facts"
+        - kolla_action is defined
+        - kolla_action == "precheck"
+
     - name: Group hosts to determine when using --limit
       group_by:
         key: "all_using_limit_{{ (ansible_play_batch | length) != (groups['all'] | length) }}"
@@ -49,4 +56,14 @@
       # We gathered facts for all hosts in the batch during the first play.
       when:
         - not hostvars[item].ansible_facts
+
+    - name: Gather package facts
+      package_facts:
+      delegate_facts: True
+      delegate_to: "{{ item }}"
+      with_items: "{{ delegate_hosts }}"
+      when:
+        - "'packages' not in hostvars[item].ansible_facts"
+        - kolla_action is defined
+        - "kolla_action == 'precheck'"
   tags: always
diff --git a/ansible/module_utils/kolla_docker_worker.py b/ansible/module_utils/kolla_docker_worker.py
index ebff465c1..8c365ca45 100644
--- a/ansible/module_utils/kolla_docker_worker.py
+++ b/ansible/module_utils/kolla_docker_worker.py
@@ -19,6 +19,7 @@ import json
 import os
 import shlex
 
+from ansible.module_utils.kolla_systemd_worker import SystemdWorker
 from distutils.version import StrictVersion
 
 COMPARE_CONFIG_CMD = ['/usr/local/bin/kolla_set_configs', '--check']
@@ -50,6 +51,8 @@ class DockerWorker(object):
         self._cgroupns_mode_supported = (
             StrictVersion(self.dc._version) >= StrictVersion('1.41'))
 
+        self.systemd = SystemdWorker(self.params)
+
     def generate_tls(self):
         tls = {'verify': self.params.get('tls_verify')}
         tls_cert = self.params.get('tls_cert'),
@@ -110,7 +113,8 @@ class DockerWorker(object):
         container = self.check_container()
         if (not container or
                 self.check_container_differs() or
-                self.compare_config()):
+                self.compare_config() or
+                self.systemd.check_unit_change()):
             self.changed = True
         return self.changed
 
@@ -469,6 +473,7 @@ class DockerWorker(object):
         self.changed = old_image_id != new_image_id
 
     def remove_container(self):
+        self.changed |= self.systemd.remove_unit_file()
         if self.check_container():
             self.changed = True
             # NOTE(jeffrey4l): in some case, docker failed to remove container
@@ -575,18 +580,6 @@ class DockerWorker(object):
             dimensions = self.parse_dimensions(dimensions)
             options.update(dimensions)
 
-        restart_policy = self.params.get('restart_policy')
-
-        if restart_policy is not None:
-            restart_policy = {'Name': restart_policy}
-            # NOTE(Jeffrey4l): MaximumRetryCount is only needed for on-failure
-            # policy
-            if restart_policy['Name'] == 'on-failure':
-                retries = self.params.get('restart_retries')
-                if retries is not None:
-                    restart_policy['MaximumRetryCount'] = retries
-            options['restart_policy'] = restart_policy
-
         if binds:
             options['binds'] = binds
 
@@ -599,6 +592,11 @@ class DockerWorker(object):
             if cgroupns_mode is not None:
                 host_config['CgroupnsMode'] = cgroupns_mode
 
+        # detached containers should only log to journald
+        if self.params.get('detach'):
+            options['log_config'] = docker.types.LogConfig(
+                type=docker.types.LogConfig.types.NONE)
+
         return host_config
 
     def _inject_env_var(self, environment_info):
@@ -637,6 +635,8 @@ class DockerWorker(object):
         self.changed = True
         options = self.build_container_options()
         self.dc.create_container(**options)
+        if self.params.get('restart_policy') != 'no':
+            self.changed |= self.systemd.create_unit_file()
 
     def recreate_or_restart_container(self):
         self.changed = True
@@ -678,7 +678,15 @@ class DockerWorker(object):
 
         if not container['Status'].startswith('Up '):
             self.changed = True
-            self.dc.start(container=self.params.get('name'))
+            if self.params.get('restart_policy') == 'no':
+                self.dc.start(container=self.params.get('name'))
+            else:
+                self.systemd.create_unit_file()
+                if not self.systemd.start():
+                    self.module.fail_json(
+                        changed=True,
+                        msg="Container timed out",
+                        **self.check_container())
 
         # We do not want to detach so we wait around for container to exit
         if not self.params.get('detach'):
@@ -806,7 +814,11 @@ class DockerWorker(object):
                     msg="No such container: {} to stop".format(name))
         elif not container['Status'].startswith('Exited '):
             self.changed = True
-            self.dc.stop(name, timeout=graceful_timeout)
+            if self.params.get('restart_policy') != 'no':
+                self.systemd.create_unit_file()
+                self.systemd.stop()
+            else:
+                self.dc.stop(name, timeout=graceful_timeout)
 
     def stop_and_remove_container(self):
         container = self.check_container()
@@ -825,8 +837,13 @@ class DockerWorker(object):
                 msg="No such container: {}".format(name))
         else:
             self.changed = True
-            self.dc.stop(name, timeout=graceful_timeout)
-            self.dc.start(name)
+            self.systemd.create_unit_file()
+
+            if not self.systemd.restart():
+                self.module.fail_json(
+                    changed=True,
+                    msg="Container timed out",
+                    **self.check_container())
 
     def create_volume(self):
         if not self.check_volume():
diff --git a/ansible/module_utils/kolla_systemd_worker.py b/ansible/module_utils/kolla_systemd_worker.py
new file mode 100644
index 000000000..2d8a59d1c
--- /dev/null
+++ b/ansible/module_utils/kolla_systemd_worker.py
@@ -0,0 +1,204 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from string import Template
+from time import sleep
+
+import dbus
+
+
+TEMPLATE = '''# ${service_name}
+# autogenerated by Kolla-Ansible
+
+[Unit]
+Description=docker ${service_name}
+After=docker.service
+Requires=docker.service
+StartLimitIntervalSec=${restart_timeout}
+StartLimitBurst=${restart_retries}
+
+[Service]
+ExecStart=/usr/bin/docker start -a ${name}
+ExecStop=/usr/bin/docker stop ${name} -t ${graceful_timeout}
+Restart=${restart_policy}
+RestartSec=${restart_duration}
+
+[Install]
+WantedBy=multi-user.target
+'''
+
+
+class SystemdWorker(object):
+    def __init__(self, params):
+        name = params.get('name', None)
+
+        # systemd is not needed
+        if not name:
+            return None
+
+        restart_policy = params.get('restart_policy', 'no')
+        if restart_policy == 'unless-stopped':
+            restart_policy = 'always'
+
+        # NOTE(hinermar): duration * retries should be less than timeout
+        # otherwise service will indefinitely try to restart.
+        # Also, correct timeout and retries values should probably be
+        # checked at the module level inside kolla_docker.py
+        restart_timeout = params.get('client_timeout', 120)
+        restart_retries = params.get('restart_retries', 10)
+        restart_duration = (restart_timeout // restart_retries) - 1
+
+        # container info
+        self.container_dict = dict(
+            name=name,
+            service_name='kolla-' + name + '-container.service',
+            engine='docker',
+            deps='docker.service',
+            graceful_timeout=params.get('graceful_timeout'),
+            restart_policy=restart_policy,
+            restart_timeout=restart_timeout,
+            restart_retries=restart_retries,
+            restart_duration=restart_duration
+        )
+
+        # systemd
+        self.manager = self.get_manager()
+        self.job_mode = 'replace'
+        self.sysdir = '/etc/systemd/system/'
+
+        # templating
+        self.template = Template(TEMPLATE)
+
+    def get_manager(self):
+        sysbus = dbus.SystemBus()
+        systemd1 = sysbus.get_object(
+            'org.freedesktop.systemd1',
+            '/org/freedesktop/systemd1'
+        )
+        return dbus.Interface(systemd1, 'org.freedesktop.systemd1.Manager')
+
+    def start(self):
+        if self.perform_action(
+            'StartUnit',
+            self.container_dict['service_name'],
+            self.job_mode
+        ):
+            return self.wait_for_unit(self.container_dict['restart_timeout'])
+        return False
+
+    def restart(self):
+        if self.perform_action(
+            'RestartUnit',
+            self.container_dict['service_name'],
+            self.job_mode
+        ):
+            return self.wait_for_unit(self.container_dict['restart_timeout'])
+        return False
+
+    def stop(self):
+        return self.perform_action(
+            'StopUnit',
+            self.container_dict['service_name'],
+            self.job_mode
+        )
+
+    def reload(self):
+        return self.perform_action(
+            'Reload',
+            self.container_dict['service_name'],
+            self.job_mode
+        )
+
+    def enable(self):
+        return self.perform_action(
+            'EnableUnitFiles',
+            [self.container_dict['service_name']],
+            False,
+            True
+        )
+
+    def perform_action(self, function, *args):
+        try:
+            getattr(self.manager, function)(*args)
+            return True
+        except Exception:
+            return False
+
+    def check_unit_file(self):
+        return os.path.isfile(
+            self.sysdir + self.container_dict['service_name']
+        )
+
+    def check_unit_change(self, new_content=''):
+        if not new_content:
+            new_content = self.generate_unit_file()
+
+        if self.check_unit_file():
+            with open(
+                self.sysdir + self.container_dict['service_name'], 'r'
+            ) as f:
+                curr_content = f.read()
+
+                # return whether there was change in the unit file
+                return curr_content != new_content
+
+        return True
+
+    def generate_unit_file(self):
+        return self.template.substitute(self.container_dict)
+
+    def create_unit_file(self):
+        file_content = self.generate_unit_file()
+
+        if self.check_unit_change(file_content):
+            with open(
+                self.sysdir + self.container_dict['service_name'], 'w'
+            ) as f:
+                f.write(file_content)
+
+            self.reload()
+            self.enable()
+            return True
+
+        return False
+
+    def remove_unit_file(self):
+        if self.check_unit_file():
+            os.remove(self.sysdir + self.container_dict['service_name'])
+            self.reload()
+
+            return True
+        else:
+            return False
+
+    def get_unit_state(self):
+        unit_list = self.manager.ListUnits()
+
+        for service in unit_list:
+            if str(service[0]) == self.container_dict['service_name']:
+                return str(service[4])
+
+        return None
+
+    def wait_for_unit(self, timeout):
+        delay = 5
+        elapsed = 0
+
+        while True:
+            if self.get_unit_state() == 'running':
+                return True
+            elif elapsed > timeout:
+                return False
+            else:
+                sleep(delay)
+                elapsed += delay
diff --git a/ansible/roles/prechecks/tasks/package_checks.yml b/ansible/roles/prechecks/tasks/package_checks.yml
index 617dba5ca..ccaa52932 100644
--- a/ansible/roles/prechecks/tasks/package_checks.yml
+++ b/ansible/roles/prechecks/tasks/package_checks.yml
@@ -9,6 +9,13 @@
     - kolla_container_engine == 'docker'
   failed_when: result is failed or result.stdout is version(docker_py_version_min, '<')
 
+- name: Checking dbus-python package
+  command: "{{ ansible_facts.python.executable }} -c \"import dbus\""
+  register: dbus_present
+  changed_when: false
+  when: inventory_hostname in groups['baremetal']
+  failed_when: dbus_present is failed
+
 # NOTE(osmanlicilegi): ansible_version.full includes patch number that's useless
 # to check. as ansible_version does not provide major.minor in dict, we need to
 # set it as variable.
diff --git a/ansible/roles/prechecks/tasks/service_checks.yml b/ansible/roles/prechecks/tasks/service_checks.yml
index f6c208be1..e85cbd708 100644
--- a/ansible/roles/prechecks/tasks/service_checks.yml
+++ b/ansible/roles/prechecks/tasks/service_checks.yml
@@ -1,4 +1,11 @@
 ---
+- name: Checking if system uses systemd
+  become: true
+  assert:
+    that:
+      - "ansible_facts.service_mgr == 'systemd'"
+  when: inventory_hostname in groups['baremetal']
+
 - name: Checking Docker version
   become: true
   command: "{{ kolla_container_engine }} --version"
diff --git a/releasenotes/notes/add-systemd-container-control-b85dff9ec5fae313.yaml b/releasenotes/notes/add-systemd-container-control-b85dff9ec5fae313.yaml
new file mode 100644
index 000000000..33833f655
--- /dev/null
+++ b/releasenotes/notes/add-systemd-container-control-b85dff9ec5fae313.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds support for container state control through systemd in kolla_docker.
+    Every container logs only to journald and has it's own unit file in
+    ``/etc/systemd/system`` named **kolla-<container name>-container.service**.
+    Systemd control is implemented in new file
+    ``ansible/module_utils/kolla_systemd_worker.py``.
diff --git a/tests/test_kolla_docker.py b/tests/test_kolla_docker.py
index d83cd5f5d..727caf9f5 100644
--- a/tests/test_kolla_docker.py
+++ b/tests/test_kolla_docker.py
@@ -25,14 +25,19 @@ from docker import errors as docker_error
 from docker.types import Ulimit
 from oslotest import base
 
+sys.modules['dbus'] = mock.MagicMock()
+
 this_dir = os.path.dirname(sys.modules[__name__].__file__)
 ansible_dir = os.path.join(this_dir, '..', 'ansible')
 kolla_docker_file = os.path.join(ansible_dir,
                                  'library', 'kolla_docker.py')
 docker_worker_file = os.path.join(ansible_dir,
                                   'module_utils', 'kolla_docker_worker.py')
+systemd_worker_file = os.path.join(ansible_dir,
+                                   'module_utils', 'kolla_systemd_worker.py')
 kd = imp.load_source('kolla_docker', kolla_docker_file)
 dwm = imp.load_source('kolla_docker_worker', docker_worker_file)
+swm = imp.load_source('kolla_systemd_worker', systemd_worker_file)
 
 
 class ModuleArgsTest(base.BaseTestCase):
@@ -157,6 +162,7 @@ FAKE_DATA = {
         'remove_on_exit': True,
         'volumes': None,
         'tty': False,
+        'client_timeout': 120,
     },
 
     'images': [
@@ -213,7 +219,8 @@ def get_DockerWorker(mod_param, docker_api_version='1.40'):
     module.params = mod_param
     with mock.patch("docker.APIClient") as MockedDockerClientClass:
         MockedDockerClientClass.return_value._version = docker_api_version
-        dw = kd.DockerWorker(module)
+        dw = dwm.DockerWorker(module)
+        dw.systemd = mock.MagicMock()
         return dw
 
 
@@ -400,11 +407,10 @@ class TestContainer(base.BaseTestCase):
         self.dw.dc.containers = mock.MagicMock(
             return_value=self.fake_data['containers'])
         self.dw.check_container_differs = mock.MagicMock(return_value=False)
-        self.dw.dc.start = mock.MagicMock()
         self.dw.start_container()
         self.assertTrue(self.dw.changed)
-        self.dw.dc.start.assert_called_once_with(
-            container=self.fake_data['params'].get('name'))
+        self.dw.dc.start.assert_not_called()
+        self.dw.systemd.start.assert_called_once()
 
     def test_start_container_no_detach(self):
         self.fake_data['params'].update({'name': 'my_container',
@@ -425,12 +431,58 @@ class TestContainer(base.BaseTestCase):
         self.dw.dc.logs.assert_has_calls([
             mock.call(name, stdout=True, stderr=False),
             mock.call(name, stdout=False, stderr=True)])
-        self.dw.dc.stop.assert_called_once_with(name, timeout=10)
+        self.dw.systemd.stop.assert_called_once_with()
         self.dw.dc.remove_container.assert_called_once_with(
             container=name, force=True)
         expected = {'rc': 0, 'stdout': 'fake stdout', 'stderr': 'fake stderr'}
         self.assertEqual(expected, self.dw.result)
 
+    def test_start_container_no_systemd(self):
+        self.fake_data['params'].update({'name': 'my_container',
+                                         'restart_policy': 'no',
+                                         'auth_username': 'fake_user',
+                                         'auth_password': 'fake_psw',
+                                         'auth_registry': 'myrepo/myapp',
+                                         'auth_email': 'fake_mail@foogle.com'})
+        self.dw = get_DockerWorker(self.fake_data['params'])
+        self.dw.dc.images = mock.MagicMock(
+            return_value=self.fake_data['images'])
+        self.fake_data['containers'][0].update(
+            {'Status': 'Exited 2 days ago'})
+        self.dw.dc.containers = mock.MagicMock(
+            return_value=self.fake_data['containers'])
+        self.dw.check_container_differs = mock.MagicMock(return_value=False)
+        self.dw.dc.start = mock.MagicMock()
+        self.dw.start_container()
+        self.assertTrue(self.dw.changed)
+        self.dw.dc.start.assert_called_once_with(
+            container=self.fake_data['params']['name']
+        )
+        self.dw.systemd.start.assert_not_called()
+
+    def test_start_container_systemd_start_fail(self):
+        self.fake_data['params'].update({'name': 'my_container',
+                                         'auth_username': 'fake_user',
+                                         'auth_password': 'fake_psw',
+                                         'auth_registry': 'myrepo/myapp',
+                                         'auth_email': 'fake_mail@foogle.com'})
+        self.dw = get_DockerWorker(self.fake_data['params'])
+        self.dw.dc.images = mock.MagicMock(
+            return_value=self.fake_data['images'])
+        self.fake_data['containers'][0].update(
+            {'Status': 'Exited 2 days ago'})
+        self.dw.dc.containers = mock.MagicMock(
+            return_value=self.fake_data['containers'])
+        self.dw.check_container_differs = mock.MagicMock(return_value=False)
+        self.dw.systemd.start = mock.Mock(return_value=False)
+        self.dw.start_container()
+        self.assertTrue(self.dw.changed)
+        self.dw.dc.start.assert_not_called()
+        self.dw.systemd.start.assert_called_once()
+        self.dw.module.fail_json.assert_called_once_with(
+            changed=True, msg='Container timed out',
+            **self.fake_data['containers'][0])
+
     def test_stop_container(self):
         self.dw = get_DockerWorker({'name': 'my_container',
                                     'action': 'stop_container'})
@@ -439,7 +491,22 @@ class TestContainer(base.BaseTestCase):
 
         self.assertTrue(self.dw.changed)
         self.dw.dc.containers.assert_called_once_with(all=True)
-        self.dw.dc.stop.assert_called_once_with('my_container', timeout=10)
+        self.dw.systemd.stop.assert_called_once()
+        self.dw.dc.stop.assert_not_called()
+        self.dw.module.fail_json.assert_not_called()
+
+    def test_stop_container_no_systemd(self):
+        self.dw = get_DockerWorker({'name': 'my_container',
+                                    'action': 'stop_container',
+                                    'restart_policy': 'no'})
+        self.dw.dc.containers.return_value = self.fake_data['containers']
+        self.dw.stop_container()
+
+        self.assertTrue(self.dw.changed)
+        self.dw.dc.containers.assert_called_once_with(all=True)
+        self.dw.systemd.stop.assert_not_called()
+        self.dw.dc.stop.assert_called_once_with(
+            'my_container', timeout=10)
         self.dw.module.fail_json.assert_not_called()
 
     def test_stop_container_already_stopped(self):
@@ -485,7 +552,7 @@ class TestContainer(base.BaseTestCase):
 
         self.assertTrue(self.dw.changed)
         self.dw.dc.containers.assert_called_with(all=True)
-        self.dw.dc.stop.assert_called_once_with('my_container', timeout=10)
+        self.dw.systemd.stop.assert_called_once()
         self.dw.dc.remove_container.assert_called_once_with(
             container='my_container', force=True)
 
@@ -497,7 +564,7 @@ class TestContainer(base.BaseTestCase):
 
         self.assertFalse(self.dw.changed)
         self.dw.dc.containers.assert_called_with(all=True)
-        self.assertFalse(self.dw.dc.stop.called)
+        self.assertFalse(self.dw.systemd.stop.called)
         self.assertFalse(self.dw.dc.remove_container.called)
 
     def test_restart_container(self):
@@ -512,9 +579,7 @@ class TestContainer(base.BaseTestCase):
 
         self.assertTrue(self.dw.changed)
         self.dw.dc.containers.assert_called_once_with(all=True)
-        self.dw.dc.inspect_container.assert_called_once_with('my_container')
-        self.dw.dc.stop.assert_called_once_with('my_container', timeout=10)
-        self.dw.dc.start.assert_called_once_with('my_container')
+        self.dw.systemd.restart.assert_called_once_with()
 
     def test_restart_container_not_exists(self):
         self.dw = get_DockerWorker({'name': 'fake-container',
@@ -527,6 +592,24 @@ class TestContainer(base.BaseTestCase):
         self.dw.module.fail_json.assert_called_once_with(
             msg="No such container: fake-container")
 
+    def test_restart_container_systemd_timeout(self):
+        self.dw = get_DockerWorker({'name': 'my_container',
+                                    'action': 'restart_container'})
+        self.dw.dc.containers.return_value = self.fake_data['containers']
+        self.fake_data['container_inspect'].update(
+            self.fake_data['containers'][0])
+        self.dw.dc.inspect_container.return_value = (
+            self.fake_data['container_inspect'])
+        self.dw.systemd.restart = mock.Mock(return_value=False)
+        self.dw.restart_container()
+
+        self.assertTrue(self.dw.changed)
+        self.dw.dc.containers.assert_called_with(all=True)
+        self.dw.systemd.restart.assert_called_once_with()
+        self.dw.module.fail_json.assert_called_once_with(
+            changed=True, msg="Container timed out",
+            **self.fake_data['containers'][0])
+
     def test_remove_container(self):
         self.dw = get_DockerWorker({'name': 'my_container',
                                     'action': 'remove_container'})
@@ -1576,3 +1659,213 @@ class TestAttrComp(base.BaseTestCase):
         self.dw = get_DockerWorker(self.fake_data['params'])
         self.assertIsNone(self.dw.parse_healthcheck(
                           self.fake_data['params']['healthcheck']))
+
+
+class TestSystemd(base.BaseTestCase):
+    def setUp(self) -> None:
+        super(TestSystemd, self).setUp()
+        self.params_dict = dict(
+            name='test',
+            restart_policy='no',
+            client_timeout=120,
+            restart_retries=10,
+            graceful_timeout=15
+        )
+        swm.sleep = mock.Mock()
+        self.sw = swm.SystemdWorker(self.params_dict)
+
+    def test_manager(self):
+        self.assertIsNotNone(self.sw)
+        self.assertIsNotNone(self.sw.manager)
+
+    def test_start(self):
+        self.sw.perform_action = mock.Mock(return_value=True)
+        self.sw.wait_for_unit = mock.Mock(return_value=True)
+
+        self.sw.start()
+        self.sw.perform_action.assert_called_once_with(
+            'StartUnit',
+            'kolla-test-container.service',
+            'replace'
+        )
+
+    def test_restart(self):
+        self.sw.perform_action = mock.Mock(return_value=True)
+        self.sw.wait_for_unit = mock.Mock(return_value=True)
+
+        self.sw.restart()
+        self.sw.perform_action.assert_called_once_with(
+            'RestartUnit',
+            'kolla-test-container.service',
+            'replace'
+        )
+
+    def test_stop(self):
+        self.sw.perform_action = mock.Mock(return_value=True)
+
+        self.sw.stop()
+        self.sw.perform_action.assert_called_once_with(
+            'StopUnit',
+            'kolla-test-container.service',
+            'replace'
+        )
+
+    def test_reload(self):
+        self.sw.perform_action = mock.Mock(return_value=True)
+
+        self.sw.reload()
+        self.sw.perform_action.assert_called_once_with(
+            'Reload',
+            'kolla-test-container.service',
+            'replace'
+        )
+
+    def test_enable(self):
+        self.sw.perform_action = mock.Mock(return_value=True)
+
+        self.sw.enable()
+        self.sw.perform_action.assert_called_once_with(
+            'EnableUnitFiles',
+            ['kolla-test-container.service'],
+            False,
+            True
+        )
+
+    def test_check_unit_change(self):
+        self.sw.generate_unit_file = mock.Mock()
+        self.sw.check_unit_file = mock.Mock(return_value=True)
+        open_mock = mock.mock_open(read_data='test data')
+        return_val = None
+
+        with mock.patch('builtins.open', open_mock, create=True):
+            return_val = self.sw.check_unit_change('test data')
+
+        self.assertFalse(return_val)
+        self.sw.generate_unit_file.assert_not_called()
+        open_mock.assert_called_with(
+            '/etc/systemd/system/kolla-test-container.service',
+            'r'
+        )
+        open_mock.return_value.read.assert_called_once()
+
+    def test_check_unit_change_diff(self):
+        self.sw.generate_unit_file = mock.Mock()
+        self.sw.check_unit_file = mock.Mock(return_value=True)
+        open_mock = mock.mock_open(read_data='new data')
+        return_val = None
+
+        with mock.patch('builtins.open', open_mock, create=True):
+            return_val = self.sw.check_unit_change('old data')
+
+        self.assertTrue(return_val)
+        self.sw.generate_unit_file.assert_not_called()
+        open_mock.assert_called_with(
+            '/etc/systemd/system/kolla-test-container.service',
+            'r'
+        )
+        open_mock.return_value.read.assert_called_once()
+
+    @mock.patch(
+        'kolla_systemd_worker.TEMPLATE',
+        """${name}, ${restart_policy},
+        ${graceful_timeout}, ${restart_timeout},
+        ${restart_retries}"""
+    )
+    def test_generate_unit_file(self):
+        self.sw = swm.SystemdWorker(self.params_dict)
+        p = self.params_dict
+        ref_string = f"""{p.get('name')}, {p.get('restart_policy')},
+        {p.get('graceful_timeout')}, {p.get('client_timeout')},
+        {p.get('restart_retries')}"""
+
+        ret_string = self.sw.generate_unit_file()
+
+        self.assertEqual(ref_string, ret_string)
+
+    def test_create_unit_file(self):
+        self.sw.generate_unit_file = mock.Mock(return_value='test data')
+        self.sw.check_unit_change = mock.Mock(return_value=True)
+        self.sw.reload = mock.Mock()
+        self.sw.enable = mock.Mock()
+        open_mock = mock.mock_open()
+        return_val = None
+
+        with mock.patch('builtins.open', open_mock, create=True):
+            return_val = self.sw.create_unit_file()
+
+        self.assertTrue(return_val)
+        open_mock.assert_called_with(
+            '/etc/systemd/system/kolla-test-container.service',
+            'w'
+        )
+        open_mock.return_value.write.assert_called_once_with('test data')
+        self.sw.reload.assert_called_once()
+        self.sw.enable.assert_called_once()
+
+    def test_create_unit_file_no_change(self):
+        self.sw.generate_unit_file = mock.Mock()
+        self.sw.check_unit_change = mock.Mock(return_value=False)
+        self.sw.reload = mock.Mock()
+        self.sw.enable = mock.Mock()
+        open_mock = mock.mock_open()
+
+        return_val = self.sw.create_unit_file()
+
+        self.assertFalse(return_val)
+        open_mock.assert_not_called()
+        self.sw.reload.assert_not_called()
+        self.sw.enable.assert_not_called()
+
+    def test_remove_unit_file(self):
+        self.sw.check_unit_file = mock.Mock(return_value=True)
+        os.remove = mock.Mock()
+        self.sw.reload = mock.Mock()
+
+        return_val = self.sw.remove_unit_file()
+
+        self.assertTrue(return_val)
+        os.remove.assert_called_once_with(
+            '/etc/systemd/system/kolla-test-container.service'
+        )
+        self.sw.reload.assert_called_once()
+
+    def test_get_unit_state(self):
+        unit_list = [
+            ('foo.service', '', 'loaded', 'active', 'exited'),
+            ('kolla-test-container.service', '', 'loaded', 'active', 'running')
+        ]
+        self.sw.manager.ListUnits = mock.Mock(return_value=unit_list)
+
+        state = self.sw.get_unit_state()
+
+        self.sw.manager.ListUnits.assert_called_once()
+        self.assertEqual('running', state)
+
+    def test_get_unit_state_not_exist(self):
+        unit_list = [
+            ('foo.service', '', 'loaded', 'active', 'exited'),
+            ('bar.service', '', 'loaded', 'active', 'running')
+        ]
+        self.sw.manager.ListUnits = mock.Mock(return_value=unit_list)
+
+        state = self.sw.get_unit_state()
+
+        self.sw.manager.ListUnits.assert_called_once()
+        self.assertIsNone(state)
+
+    def test_wait_for_unit(self):
+        self.sw.get_unit_state = mock.Mock()
+        self.sw.get_unit_state.side_effect = ['starting', 'running']
+
+        result = self.sw.wait_for_unit(10)
+
+        self.assertTrue(result)
+
+    def test_wait_for_unit_timeout(self):
+        self.sw.get_unit_state = mock.Mock()
+        self.sw.get_unit_state.side_effect = [
+            'starting', 'starting', 'failed', 'failed']
+
+        result = self.sw.wait_for_unit(10)
+
+        self.assertFalse(result)
diff --git a/tools/cleanup-containers b/tools/cleanup-containers
index 16f7ccf44..493bdcac6 100755
--- a/tools/cleanup-containers
+++ b/tools/cleanup-containers
@@ -30,7 +30,9 @@ echo "Removing ovs bridge..."
 fi
 
 echo "Stopping containers..."
-(sudo docker stop -t 2 ${containers_to_kill} 2>&1) > /dev/null
+for container in ${containers_to_kill}; do
+sudo systemctl stop kolla-${container}-container.service
+done
 
 echo "Removing containers..."
 (sudo docker rm -v -f ${containers_to_kill} 2>&1) > /dev/null
@@ -46,4 +48,8 @@ echo "Removing volumes..."
 echo "Removing link of kolla_log volume..."
 (sudo rm -f /var/log/kolla 2>&1) > /dev/null
 
+echo "Removing unit files..."
+sudo rm -f /etc/systemd/system/kolla-*-container.service
+sudo systemctl daemon-reload
+
 echo "All cleaned up!"
diff --git a/tox.ini b/tox.ini
index 11cfe70f3..ec9a22fae 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,7 +30,7 @@ setenv = VIRTUAL_ENV={envdir}
          NOSE_COVER_BRANCHES=1
          NOSE_COVER_HTML=1
          NOSE_COVER_HTML_DIR={toxinidir}/cover
-         PYTHON=coverage run --source kolla_ansible,ansible/action_plugins,ansible/library,ansible/roles/keystone/files/ --parallel-mode
+         PYTHON=coverage run --source kolla_ansible,ansible/action_plugins,ansible/library,ansible/module_utils,ansible/roles/keystone/files/ --parallel-mode
 commands =
   bash {toxinidir}/tests/link-module-utils.sh {toxinidir} {envsitepackagesdir}
   stestr run {posargs}