Merge "Choose node with largest seqno number for mariadb recovery"

3 files changed, 58 insertions, 2 deletions

diff --git a/ansible/roles/mariadb/tasks/recover_cluster.yml b/ansible/roles/mariadb/tasks/recover_cluster.yml
index 2e0fcf5b4..f45b907cb 100644
--- a/ansible/roles/mariadb/tasks/recover_cluster.yml
+++ b/ansible/roles/mariadb/tasks/recover_cluster.yml
@@ -14,6 +14,60 @@
   when: container_state.Running | bool
   any_errors_fatal: True
 
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
+- block:
+  - name: Copying grastate.dat file from mariadb container
+    command: docker cp mariadb:/var/lib/mysql/grastate.dat /tmp/kolla_mariadb_grastate.dat
+    changed_when: false
+
+  - name: Print the content of grastate.dat file
+    command: cat /tmp/kolla_mariadb_grastate.dat
+    register: cat_grastate
+    changed_when: false
+
+  - name: Registering mariadb seqno variable
+    set_fact:
+      seqno: "{{ (cat_grastate.stdout|from_yaml).seqno }}"
+    changed_when: false
+
+  - name: Comparing seqno value on all mariadb hosts
+    shell: "if [[ {{ hostvars[inventory_hostname]['seqno'] }} -lt {{ hostvars[item]['seqno'] }} ]]; then echo {{ hostvars[item]['seqno'] }}; fi"
+    with_items: "{{ groups['mariadb'] }}"
+    changed_when: false
+    register: seqno_compare
+
+  - name: Writing hostname of host with the largest seqno to temp file
+    local_action: copy content={{ inventory_hostname }} dest=/tmp/kolla_mariadb_recover_inventory_name mode=0644
+    changed_when: false
+    when: seqno_compare.results | map(attribute='stdout') | join('') == ""
+
+  - name: Registering mariadb_recover_inventory_name from temp file
+    set_fact:
+      mariadb_recover_inventory_name: "{{ lookup('file', '/tmp/kolla_mariadb_recover_inventory_name') }}"
+  when:
+    - mariadb_recover_inventory_name is not defined
+
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
 - name: Starting first MariaDB container
   kolla_docker:
     action: "start_container"
diff --git a/ansible/roles/mariadb/tasks/start.yml b/ansible/roles/mariadb/tasks/start.yml
index 3eff6ebee..f71ad37a5 100644
--- a/ansible/roles/mariadb/tasks/start.yml
+++ b/ansible/roles/mariadb/tasks/start.yml
@@ -12,7 +12,9 @@
       - "kolla_logs:/var/log/kolla/"
   when: delegate_host != 'None' or
         ( groups['mariadb'] | length ) == 1 or
-        ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is not defined and inventory_hostname != groups['mariadb'][0] ) or
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is defined and inventory_hostname != mariadb_recover_inventory_name )
+
 
 # TODO(jeffrey4l), remove the task check when the wair_for bug is fixed
 # https://github.com/ansible/ansible-modules-core/issues/2788
diff --git a/tools/kolla-ansible b/tools/kolla-ansible
index 21b3c8263..97a7c358c 100755
--- a/tools/kolla-ansible
+++ b/tools/kolla-ansible
@@ -152,7 +152,7 @@ case "$1" in
         ;;
 (mariadb_recovery)
         ACTION="Attempting to restart mariadb cluster"
-        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"
+        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy -e common_run=true"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
         ;;
 (destroy)