Build images with healthcheck scripts for core services

blueprint container-health-check

Now docker supports healthcheck directive[0] in Dockerfile,
which is helpful for checking container aliveness.

Base for this patch were healthcheck scripts from tripleo[1].

[0] https://docs.docker.com/engine/reference/builder/#healthcheck
[1] https://github.com/openstack/tripleo-common/tree/master/healthcheck

Change-Id: Ifc32713cd4d6b45c74855c0ebf5bb65784701d44

10 files changed, 92 insertions, 0 deletions

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index 590bec89e..793471e77 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -484,6 +484,15 @@ ENTRYPOINT ["dumb-init", "--single-child", "--"]
 
 {% endif %}
 
+{% if docker_healthchecks %}
+{% block healthcheck_installation %}
+
+COPY healthcheck_curl healthcheck_filemod healthcheck_listen healthcheck_port healthcheck_socket /usr/local/bin/
+RUN chmod 755 /usr/local/bin/healthcheck_*
+
+{% endblock %}
+{% endif %}
+
 RUN touch /usr/local/bin/kolla_extend_start \
     && chmod 755 /usr/local/bin/kolla_start /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_set_configs /usr/local/bin/kolla_copy_cacerts \
     && chmod 440 /etc/sudoers \
diff --git a/docker/base/healthcheck_curl b/docker/base/healthcheck_curl
new file mode 100755
index 000000000..2ef8538ba
--- /dev/null
+++ b/docker/base/healthcheck_curl
@@ -0,0 +1,12 @@
+#!/bin/bash
+: ${HEALTHCHECK_CURL_MAX_TIME:=10}
+: ${HEALTHCHECK_CURL_USER_AGENT:=curl-healthcheck}
+: ${HEALTHCHECK_CURL_WRITE_OUT:='\n%{http_code} %{remote_ip}:%{remote_port} %{time_total} seconds\n'}
+: ${HEALTHCHECK_CURL_OUTPUT:='/dev/null'}
+
+export NSS_SDB_USE_CACHE=no
+curl -g -k -q -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \
+    --max-time "${HEALTHCHECK_CURL_MAX_TIME}" \
+    --user-agent "${HEALTHCHECK_CURL_USER_AGENT}" \
+    --write-out "${HEALTHCHECK_CURL_WRITE_OUT}" \
+    "$@" || exit 1
diff --git a/docker/base/healthcheck_filemod b/docker/base/healthcheck_filemod
new file mode 100755
index 000000000..007643d9b
--- /dev/null
+++ b/docker/base/healthcheck_filemod
@@ -0,0 +1,15 @@
+#!/bin/bash
+file_path=$1
+limit_seconds=$2
+
+# if the file doesn't exist, return 1
+if [ ! -f $file_path ]; then
+    echo "${file_path} does not exist for file modification check"
+    exit 1
+fi
+curr_time=$(date +%s)
+last_mod=$(stat -c '%Y' $file_path)
+limit_epoch=$(( curr_time-limit_seconds ))
+if [ ${limit_epoch} -gt ${last_mod} ]; then
+    exit 1
+fi
diff --git a/docker/base/healthcheck_listen b/docker/base/healthcheck_listen
new file mode 100755
index 000000000..70273339c
--- /dev/null
+++ b/docker/base/healthcheck_listen
@@ -0,0 +1,8 @@
+#!/bin/bash
+process=$1
+
+shift 1
+args=$@
+ports=${args// /|}
+pids=$(pgrep -d '|' -f $process)
+ss -lnp | grep -qE ":($ports).*,pid=($pids),"
diff --git a/docker/base/healthcheck_port b/docker/base/healthcheck_port
new file mode 100755
index 000000000..9531bd77a
--- /dev/null
+++ b/docker/base/healthcheck_port
@@ -0,0 +1,8 @@
+#!/bin/bash
+process=$1
+
+shift 1
+args=$@
+ports=${args// /|}
+pids=$(pgrep -d '|' -f $process)
+ss -ntp | grep -qE ":($ports).*,pid=($pids),"
diff --git a/docker/base/healthcheck_socket b/docker/base/healthcheck_socket
new file mode 100755
index 000000000..cd65bae69
--- /dev/null
+++ b/docker/base/healthcheck_socket
@@ -0,0 +1,10 @@
+#!/bin/bash
+process=$1
+socket=$2
+
+# lsof truncate command name to 15 characters and this behaviour
+# cannot be disabled
+if [ ${#process} -gt 15 ] ; then
+    process=${process:0:15}
+fi
+lsof -Fc -Ua $socket | grep -q "c$process"
diff --git a/docker/mariadb/Dockerfile.j2 b/docker/mariadb/Dockerfile.j2
index 491349459..d958b03b0 100644
--- a/docker/mariadb/Dockerfile.j2
+++ b/docker/mariadb/Dockerfile.j2
@@ -50,6 +50,15 @@ RUN mkdir -p /var/run/mysqld && chown mysql /var/run/mysqld && chmod 755 /var/ru
 COPY backup.sh /usr/local/bin/kolla_mariadb_backup.sh
 RUN chmod 755 /usr/local/bin/kolla_mariadb_backup.sh
 
+{% if docker_healthchecks %}
+{% block healthcheck_installation %}
+
+COPY healthcheck_mariadb /usr/local/bin/healthcheck_mariadb
+RUN chmod 755 /usr/local/bin/healthcheck_mariadb
+
+{% endblock %}
+{% endif %}
+
 {% if use_dumb_init %}
 {% block mariadb_entrypoint %}
 # NOTE(mgoddard): Override the dumb-init arguments to avoid passing
diff --git a/docker/mariadb/healthcheck_mariadb b/docker/mariadb/healthcheck_mariadb
new file mode 100755
index 000000000..b79b19bb4
--- /dev/null
+++ b/docker/mariadb/healthcheck_mariadb
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+MYSQL_USERNAME="${MYSQL_USERNAME:=-haproxy}"
+MYSQL_TIMEOUT=10
+
+MYSQL_CMDLINE="mysql -nNE --connect-timeout=${MYSQL_TIMEOUT} -u ${MYSQL_USERNAME}"
+
+WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state_comment';")
+
+if [[ "${WSREP_STATUS}" == "Synced" ]]
+then
+    echo "MariaDB Galera Cluster Node is synced."
+    exit 0
+else
+    echo "MariaDB Galera Cluster Node is NOT synced"
+    exit 0
+fi
diff --git a/kolla/common/config.py b/kolla/common/config.py
index 689631010..6b5302f46 100755
--- a/kolla/common/config.py
+++ b/kolla/common/config.py
@@ -255,6 +255,8 @@ _CLI_OPTS = [
                help='OpenStack release for building kolla-toolbox'),
     cfg.StrOpt('openstack-branch', default='master',
                help='Branch for source images'),
+    cfg.BoolOpt('docker-healthchecks', default=True,
+                help='Add Kolla docker healthcheck scripts in the image')
 ]
 
 _BASE_OPTS = [
diff --git a/kolla/image/build.py b/kolla/image/build.py
index b29450edd..2750d1a43 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -718,6 +718,7 @@ class KollaWorker(object):
             self.debian_arch = 'ppc64el'
         self.images = list()
         self.openstack_release = conf.openstack_release
+        self.docker_healthchecks = conf.docker_healthchecks
         rpm_setup_config = ([repo_file for repo_file in
                              conf.rpm_setup_config if repo_file is not None])
         self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
@@ -960,6 +961,7 @@ class KollaWorker(object):
                       'use_dumb_init': self.use_dumb_init,
                       'base_package_type': self.base_package_type,
                       'debian_arch': self.debian_arch,
+                      'docker_healthchecks': self.docker_healthchecks,
                       'supported_distro_release': supported_distro_release,
                       'install_metatype': self.install_metatype,
                       'image_prefix': self.image_prefix,