Make galeradb bootstraping robust

Currently, there are arbitrary wait for mariadb service startup.
However, this leads to nondeterministic results in the current
workflow. This patch tries to make the workflow more deterministic.

Change-Id: I3c6245cce93c7ff0d3d57cb2ae065a1ed1487769
Closes-Bug: #1491782

6 files changed, 80 insertions, 17 deletions

diff --git a/ansible/roles/mariadb/defaults/main.yml b/ansible/roles/mariadb/defaults/main.yml
index 00f49d1ba..410e86d3e 100644
--- a/ansible/roles/mariadb/defaults/main.yml
+++ b/ansible/roles/mariadb/defaults/main.yml
@@ -5,7 +5,7 @@ project_name: "mariadb"
 # Database
 ####################
 database_cluster_name: "openstack"
-
+database_max_timeout: 60
 
 ####################
 # Docker
diff --git a/ansible/roles/mariadb/tasks/bootstrap.yml b/ansible/roles/mariadb/tasks/bootstrap.yml
index 68354dd15..af7d8f308 100644
--- a/ansible/roles/mariadb/tasks/bootstrap.yml
+++ b/ansible/roles/mariadb/tasks/bootstrap.yml
@@ -44,6 +44,8 @@
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
       DB_ROOT_PASSWORD: "{{ database_password }}"
+      DB_PID_FILE: "/var/lib/mysql/{{ hostvars[inventory_hostname]['ansible_hostname'] }}-galera.pid"
+      DB_MAX_TIMEOUT: "{{ database_max_timeout }}"
     image: "{{ mariadb_image_full }}"
     name: "mariadb"
     restart_policy: "never"
@@ -52,7 +54,11 @@
       - "mariadb:/var/lib/mysql"
   when: delegate_host == 'None' and inventory_hostname == groups['mariadb'][0]
 
-- name: Sleeping for 15 seconds while the cluster starts
-  command: sleep 15
-  changed_when: False
+- name: Waiting for MariaDB service to be ready
+  command: "docker exec mariadb ls /var/lib/mysql/{{ hostvars[inventory_hostname]['ansible_hostname'] }}-galera.pid /var/lib/mysql/cluster.exists"
+  register: bootstrap_result
   when: delegate_host == 'None' and inventory_hostname == groups['mariadb'][0]
+  until: bootstrap_result|success
+  changed_when: False
+  retries: 6
+  delay: 10
diff --git a/ansible/roles/mariadb/tasks/start.yml b/ansible/roles/mariadb/tasks/start.yml
index 4682455ca..cfc12c063 100644
--- a/ansible/roles/mariadb/tasks/start.yml
+++ b/ansible/roles/mariadb/tasks/start.yml
@@ -11,3 +11,14 @@
   when: delegate_host != 'None' or
         ( groups['mariadb'] | length ) == 1 or
         ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+
+- name: Waiting for MariaDB service to be ready
+  command: "docker exec mariadb ls /var/lib/mysql/{{ hostvars[inventory_hostname]['ansible_hostname'] }}-galera.pid"
+  register: bootstrap_result
+  when: delegate_host != 'None' or
+        ( groups['mariadb'] | length ) == 1 or
+        ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+  until: bootstrap_result|success
+  changed_when: False
+  retries: 6
+  delay: 10
diff --git a/ansible/roles/mariadb/templates/galera.cnf.j2 b/ansible/roles/mariadb/templates/galera.cnf.j2
index bed806883..4e1039268 100644
--- a/ansible/roles/mariadb/templates/galera.cnf.j2
+++ b/ansible/roles/mariadb/templates/galera.cnf.j2
@@ -24,3 +24,6 @@ wsrep_sst_auth={{ database_user }}:{{ database_password }}
 wsrep_slave_threads=4
 
 max_connections=1000
+
+[server]
+pid-file=/var/lib/mysql/{{ ansible_hostname }}-galera.pid
diff --git a/docker/mariadb/extend_start.sh b/docker/mariadb/extend_start.sh
index 2ff24b73b..dcbdac2ed 100644
--- a/docker/mariadb/extend_start.sh
+++ b/docker/mariadb/extend_start.sh
@@ -2,11 +2,17 @@
 
 function bootstrap_db {
     mysqld_safe --wsrep-new-cluster &
-
-    # Waiting for deamon
-    sleep 10
+    # Wait for the mariadb server to be "Ready" before starting the security reset with a max timeout
+    TIMEOUT=${DB_MAX_TIMEOUT:-60}
+    while [ ! -f ${DB_PID_FILE} ]; do
+        if [[ ${TIMEOUT} -gt 0 ]]; then
+            let TIMEOUT-=1
+            sleep 1
+        else
+            exit 1
+        fi
+    done
     sudo -E kolla_security_reset
-
     mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
     mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
     mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown
@@ -17,7 +23,7 @@ sudo chown mysql: /var/lib/mysql
 # This catches all cases of the BOOTSTRAP variable being set, including empty
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]] && [[ ! -e /var/lib/mysql/cluster.exists ]]; then
     ARGS="--wsrep-new-cluster"
-    touch /var/lib/mysql/cluster.exists
     mysql_install_db
     bootstrap_db
+    touch /var/lib/mysql/cluster.exists
 fi
diff --git a/docker/mariadb/security_reset.expect b/docker/mariadb/security_reset.expect
index 7c720136f..af71463fb 100644
--- a/docker/mariadb/security_reset.expect
+++ b/docker/mariadb/security_reset.expect
@@ -2,20 +2,57 @@
 
 set timeout 10
 spawn mysql_secure_installation
-expect "Enter current password for root (enter for none):"
+expect {
+  timeout { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  "Enter current password for root (enter for none):"
+}
 send "\r"
-expect "Set root password?"
+expect {
+  timeout { send_user "\nFailed to get 'Set root password?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Set root password?' prompt\n"; exit 1 }
+  "Set root password?"
+}
 send "y\r"
-expect "New password:"
+expect {
+  timeout { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  "New password:"
+}
 send "$env(DB_ROOT_PASSWORD)\r"
-expect "Re-enter new password:"
+
+expect {
+  timeout { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  "Re-enter new password:"
+}
 send "$env(DB_ROOT_PASSWORD)\r"
-expect "Remove anonymous users?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  "Remove anonymous users?"
+}
 send "y\r"
-expect "Disallow root login remotely?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  "Disallow root login remotely?"
+}
 send "n\r"
-expect "Remove test database and access to it?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  "Remove test database and access to it?"
+}
 send "y\r"
-expect "Reload privilege tables now?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  "Reload privilege tables now?"
+}
 send "y\r"
 expect eof