systemd: handle running container without systemd unit

MariaDB bootstrap has a phase where the first MariaDB container
is running with Galera bootstrap - after a check that WSREP
is synced is successful - we restart the container.

The bootstrap container is named mariadb and running with
docker_restart_policy: "no" - the restarted container should be running
in systemd.

Before this patch the code created a systemd unit but it was initially
stopped - so stopping was always a success - and the container would be
killed with SIGKILL on removal (which obviously breaks MariaDB).

This patch also improves docker/systemd stops by waiting for real
unit/container stop and adds failing CI for containers that are
killed with signal 9.

Closes-Bug: #2029613

Change-Id: I0a03e509ce228a50e081fcab44d2b4831251190c

2 files changed, 12 insertions, 8 deletions

diff --git a/ansible/module_utils/kolla_docker_worker.py b/ansible/module_utils/kolla_docker_worker.py
index 72400ea20..2741b006d 100644
--- a/ansible/module_utils/kolla_docker_worker.py
+++ b/ansible/module_utils/kolla_docker_worker.py
@@ -446,11 +446,10 @@ class DockerWorker(ContainerWorker):
                     msg="No such container: {} to stop".format(name))
         elif not container['Status'].startswith('Exited '):
             self.changed = True
-            if self.params.get('restart_policy') != 'no':
-                self.systemd.create_unit_file()
-                self.systemd.stop()
-            else:
+            if not self.systemd.check_unit_file():
                 self.dc.stop(name, timeout=graceful_timeout)
+            else:
+                self.systemd.stop()
 
     def stop_and_remove_container(self):
         container = self.check_container()
diff --git a/ansible/module_utils/kolla_systemd_worker.py b/ansible/module_utils/kolla_systemd_worker.py
index 2d8a59d1c..72c50b316 100644
--- a/ansible/module_utils/kolla_systemd_worker.py
+++ b/ansible/module_utils/kolla_systemd_worker.py
@@ -106,11 +106,16 @@ class SystemdWorker(object):
         return False
 
     def stop(self):
-        return self.perform_action(
+        if self.perform_action(
             'StopUnit',
             self.container_dict['service_name'],
             self.job_mode
-        )
+        ):
+            return self.wait_for_unit(
+                self.container_dict['restart_timeout'],
+                state='dead'
+            )
+        return False
 
     def reload(self):
         return self.perform_action(
@@ -190,12 +195,12 @@ class SystemdWorker(object):
 
         return None
 
-    def wait_for_unit(self, timeout):
+    def wait_for_unit(self, timeout, state='running'):
         delay = 5
         elapsed = 0
 
         while True:
-            if self.get_unit_state() == 'running':
+            if self.get_unit_state() == state:
                 return True
             elif elapsed > timeout:
                 return False