Choose node with largest seqno number for mariadb recovery

When all mariadb nodes are stopped gracefully, mariadb galera will
write it's last executed position into the grastate.dat file. Need find
the node with largest seqno number in that file and recovery from that
node.

Closes-Bug: #1627717
Change-Id: I6e97c190eec99c966bffde0698f783e519ba14bd

3 files changed, 58 insertions, 2 deletions

diff --git a/ansible/roles/mariadb/tasks/recover_cluster.yml b/ansible/roles/mariadb/tasks/recover_cluster.yml
index 2e0fcf5b4..f45b907cb 100644
--- a/ansible/roles/mariadb/tasks/recover_cluster.yml
+++ b/ansible/roles/mariadb/tasks/recover_cluster.yml
@@ -14,6 +14,60 @@
   when: container_state.Running | bool
   any_errors_fatal: True
 
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
+- block:
+  - name: Copying grastate.dat file from mariadb container
+    command: docker cp mariadb:/var/lib/mysql/grastate.dat /tmp/kolla_mariadb_grastate.dat
+    changed_when: false
+
+  - name: Print the content of grastate.dat file
+    command: cat /tmp/kolla_mariadb_grastate.dat
+    register: cat_grastate
+    changed_when: false
+
+  - name: Registering mariadb seqno variable
+    set_fact:
+      seqno: "{{ (cat_grastate.stdout|from_yaml).seqno }}"
+    changed_when: false
+
+  - name: Comparing seqno value on all mariadb hosts
+    shell: "if [[ {{ hostvars[inventory_hostname]['seqno'] }} -lt {{ hostvars[item]['seqno'] }} ]]; then echo {{ hostvars[item]['seqno'] }}; fi"
+    with_items: "{{ groups['mariadb'] }}"
+    changed_when: false
+    register: seqno_compare
+
+  - name: Writing hostname of host with the largest seqno to temp file
+    local_action: copy content={{ inventory_hostname }} dest=/tmp/kolla_mariadb_recover_inventory_name mode=0644
+    changed_when: false
+    when: seqno_compare.results | map(attribute='stdout') | join('') == ""
+
+  - name: Registering mariadb_recover_inventory_name from temp file
+    set_fact:
+      mariadb_recover_inventory_name: "{{ lookup('file', '/tmp/kolla_mariadb_recover_inventory_name') }}"
+  when:
+    - mariadb_recover_inventory_name is not defined
+
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
 - name: Starting first MariaDB container
   kolla_docker:
     action: "start_container"
diff --git a/ansible/roles/mariadb/tasks/start.yml b/ansible/roles/mariadb/tasks/start.yml
index 3eff6ebee..f71ad37a5 100644
--- a/ansible/roles/mariadb/tasks/start.yml
+++ b/ansible/roles/mariadb/tasks/start.yml
@@ -12,7 +12,9 @@
       - "kolla_logs:/var/log/kolla/"
   when: delegate_host != 'None' or
         ( groups['mariadb'] | length ) == 1 or
-        ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is not defined and inventory_hostname != groups['mariadb'][0] ) or
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is defined and inventory_hostname != mariadb_recover_inventory_name )
+
 
 # TODO(jeffrey4l), remove the task check when the wair_for bug is fixed
 # https://github.com/ansible/ansible-modules-core/issues/2788
diff --git a/tools/kolla-ansible b/tools/kolla-ansible
index 21b3c8263..97a7c358c 100755
--- a/tools/kolla-ansible
+++ b/tools/kolla-ansible
@@ -152,7 +152,7 @@ case "$1" in
         ;;
 (mariadb_recovery)
         ACTION="Attempting to restart mariadb cluster"
-        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"
+        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy -e common_run=true"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
         ;;
 (destroy)