Fix mariadb bootstrap error

Wait until the mariadb cluster is fully functional
- check existence of mariadb.pid is insufficient because the cluster
  may not be fully prepared

Change-Id: I611b38f7dbc8032c42aee2a040fb1210b4bee7eb
closes-bug: #1614363

1 files changed, 16 insertions, 1 deletions

diff --git a/docker/mariadb/extend_start.sh b/docker/mariadb/extend_start.sh
index 629b3a98c..f3346d08c 100644
--- a/docker/mariadb/extend_start.sh
+++ b/docker/mariadb/extend_start.sh
@@ -3,8 +3,12 @@
 function bootstrap_db {
     mysqld_safe --wsrep-new-cluster &
     # Wait for the mariadb server to be "Ready" before starting the security reset with a max timeout
+    # NOTE(huikang): the location of mysql's socket file varies depending on the OS distributions.
+    # Querying the cluster status has to be executed after the existence of mysql.sock and mariadb.pid.
     TIMEOUT=${DB_MAX_TIMEOUT:-60}
-    while [[ ! -f /var/lib/mysql/mariadb.pid ]]; do
+    while [[ ! -S /var/lib/mysql/mysql.sock ]] && \
+          [[ ! -S /var/run/mysqld/mysqld.sock ]] || \
+          [[ ! -f /var/lib/mysql/mariadb.pid ]]; do
         if [[ ${TIMEOUT} -gt 0 ]]; then
             let TIMEOUT-=1
             sleep 1
@@ -12,6 +16,17 @@ function bootstrap_db {
             exit 1
         fi
     done
+    CLUSTER_READY=$(mysql -u root --exec="SHOW STATUS LIKE 'wsrep_cluster_status'" | grep Primary)
+    TIMEOUT=${DB_MAX_TIMEOUT:-60}
+    while [[ -z "${CLUSTER_READY}" ]]; do
+        if [[ ${TIMEOUT} -gt 0 ]]; then
+            let TIMEOUT-=1
+            sleep 1
+        else
+            exit 1
+        fi
+    done
+
     sudo -E kolla_security_reset
     mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
     mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"