Merge "Make galeradb bootstraping robust"

6 files changed, 80 insertions, 17 deletions

diff --git a/ansible/roles/mariadb/defaults/main.yml b/ansible/roles/mariadb/defaults/main.yml
index 00f49d1ba..410e86d3e 100644
--- a/ansible/roles/mariadb/defaults/main.yml
+++ b/ansible/roles/mariadb/defaults/main.yml
@@ -5,7 +5,7 @@ project_name: "mariadb"
 # Database
 ####################
 database_cluster_name: "openstack"
-
+database_max_timeout: 60
 
 ####################
 # Docker
diff --git a/ansible/roles/mariadb/tasks/bootstrap.yml b/ansible/roles/mariadb/tasks/bootstrap.yml
index 68354dd15..af7d8f308 100644
--- a/ansible/roles/mariadb/tasks/bootstrap.yml
+++ b/ansible/roles/mariadb/tasks/bootstrap.yml
@@ -44,6 +44,8 @@
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
       DB_ROOT_PASSWORD: "{{ database_password }}"
+      DB_PID_FILE: "/var/lib/mysql/{{ hostvars[inventory_hostname]['ansible_hostname'] }}-galera.pid"
+      DB_MAX_TIMEOUT: "{{ database_max_timeout }}"
     image: "{{ mariadb_image_full }}"
     name: "mariadb"
     restart_policy: "never"
@@ -52,7 +54,11 @@
       - "mariadb:/var/lib/mysql"
   when: delegate_host == 'None' and inventory_hostname == groups['mariadb'][0]
 
-- name: Sleeping for 15 seconds while the cluster starts
-  command: sleep 15
-  changed_when: False
+- name: Waiting for MariaDB service to be ready
+  command: "docker exec mariadb ls /var/lib/mysql/{{ hostvars[inventory_hostname]['ansible_hostname'] }}-galera.pid /var/lib/mysql/cluster.exists"
+  register: bootstrap_result
   when: delegate_host == 'None' and inventory_hostname == groups['mariadb'][0]
+  until: bootstrap_result|success
+  changed_when: False
+  retries: 6
+  delay: 10
diff --git a/ansible/roles/mariadb/tasks/start.yml b/ansible/roles/mariadb/tasks/start.yml
index 4682455ca..cfc12c063 100644
--- a/ansible/roles/mariadb/tasks/start.yml
+++ b/ansible/roles/mariadb/tasks/start.yml
@@ -11,3 +11,14 @@
   when: delegate_host != 'None' or
         ( groups['mariadb'] | length ) == 1 or
         ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+
+- name: Waiting for MariaDB service to be ready
+  command: "docker exec mariadb ls /var/lib/mysql/{{ hostvars[inventory_hostname]['ansible_hostname'] }}-galera.pid"
+  register: bootstrap_result
+  when: delegate_host != 'None' or
+        ( groups['mariadb'] | length ) == 1 or
+        ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+  until: bootstrap_result|success
+  changed_when: False
+  retries: 6
+  delay: 10
diff --git a/ansible/roles/mariadb/templates/galera.cnf.j2 b/ansible/roles/mariadb/templates/galera.cnf.j2
index bed806883..4e1039268 100644
--- a/ansible/roles/mariadb/templates/galera.cnf.j2
+++ b/ansible/roles/mariadb/templates/galera.cnf.j2
@@ -24,3 +24,6 @@ wsrep_sst_auth={{ database_user }}:{{ database_password }}
 wsrep_slave_threads=4
 
 max_connections=1000
+
+[server]
+pid-file=/var/lib/mysql/{{ ansible_hostname }}-galera.pid
diff --git a/docker/mariadb/extend_start.sh b/docker/mariadb/extend_start.sh
index 2ff24b73b..dcbdac2ed 100644
--- a/docker/mariadb/extend_start.sh
+++ b/docker/mariadb/extend_start.sh
@@ -2,11 +2,17 @@
 
 function bootstrap_db {
     mysqld_safe --wsrep-new-cluster &
-
-    # Waiting for deamon
-    sleep 10
+    # Wait for the mariadb server to be "Ready" before starting the security reset with a max timeout
+    TIMEOUT=${DB_MAX_TIMEOUT:-60}
+    while [ ! -f ${DB_PID_FILE} ]; do
+        if [[ ${TIMEOUT} -gt 0 ]]; then
+            let TIMEOUT-=1
+            sleep 1
+        else
+            exit 1
+        fi
+    done
     sudo -E kolla_security_reset
-
     mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
     mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
     mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown
@@ -17,7 +23,7 @@ sudo chown mysql: /var/lib/mysql
 # This catches all cases of the BOOTSTRAP variable being set, including empty
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]] && [[ ! -e /var/lib/mysql/cluster.exists ]]; then
     ARGS="--wsrep-new-cluster"
-    touch /var/lib/mysql/cluster.exists
     mysql_install_db
     bootstrap_db
+    touch /var/lib/mysql/cluster.exists
 fi
diff --git a/docker/mariadb/security_reset.expect b/docker/mariadb/security_reset.expect
index 7c720136f..af71463fb 100644
--- a/docker/mariadb/security_reset.expect
+++ b/docker/mariadb/security_reset.expect
@@ -2,20 +2,57 @@
 
 set timeout 10
 spawn mysql_secure_installation
-expect "Enter current password for root (enter for none):"
+expect {
+  timeout { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  "Enter current password for root (enter for none):"
+}
 send "\r"
-expect "Set root password?"
+expect {
+  timeout { send_user "\nFailed to get 'Set root password?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Set root password?' prompt\n"; exit 1 }
+  "Set root password?"
+}
 send "y\r"
-expect "New password:"
+expect {
+  timeout { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  "New password:"
+}
 send "$env(DB_ROOT_PASSWORD)\r"
-expect "Re-enter new password:"
+
+expect {
+  timeout { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  "Re-enter new password:"
+}
 send "$env(DB_ROOT_PASSWORD)\r"
-expect "Remove anonymous users?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  "Remove anonymous users?"
+}
 send "y\r"
-expect "Disallow root login remotely?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  "Disallow root login remotely?"
+}
 send "n\r"
-expect "Remove test database and access to it?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  "Remove test database and access to it?"
+}
 send "y\r"
-expect "Reload privilege tables now?"
+
+expect {
+  timeout { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  "Reload privilege tables now?"
+}
 send "y\r"
 expect eof