path: root/ansible/roles/mariadb/tasks/recover_cluster.yml

---
- name: Checking for mariadb cluster
  fail:
    msg: "MariaDB cluster was not found. Is your inventory correct?"
  when: not mariadb_cluster_exists

- name: Cleaning up temp file on localhost
  file:
    path: "{{ item }}"
    state: absent
  delegate_to: localhost
  connection: local
  changed_when: false
  check_mode: no
  run_once: true
  with_fileglob: "/tmp/kolla_mariadb_recover_inventory_name_*"

- block:
    - name: Stop MariaDB containers
      become: true
      kolla_container:
        name: "{{ mariadb_service.container_name }}"
        action: "stop_container"
        common_options: "{{ docker_common_options }}"
        ignore_missing: true

    # Run wsrep recovery with detach=false to block until completion. Use a
    # different container name to avoid the mariadb container being removed.
    - name: Run MariaDB wsrep recovery
      become: true
      kolla_container:
        action: "start_container"
        common_options: "{{ docker_common_options }}"
        detach: false
        environment:
          KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
          BOOTSTRAP_ARGS: "--wsrep-recover"
        image: "{{ mariadb_service.image }}"
        labels:
          BOOTSTRAP:
        name: mariadb_wsrep_recovery
        restart_policy: oneshot
        volumes: "{{ mariadb_service.volumes }}"

    - name: Copying MariaDB log file to /tmp
      become: true
      command: "cp {{ container_engine_volumes_path }}/kolla_logs/_data/mariadb/mariadb.log /tmp/mariadb_tmp.log"

    # Look for sequence number in logs. Format is:
    # WSREP: Recovered position: <UUID>:<seqno>.
    - name: Get MariaDB wsrep recovery seqno
      become: true
      shell: awk -F" " '/Recovered position/{seqno=$NF} END{split(seqno, a, ":"); print a[2]}' /tmp/mariadb_tmp.log
      register: wsrep_recovery_seqno

    - name: Removing MariaDB log file from /tmp
      become: true
      file:
        path: /tmp/mariadb_tmp.log
        state: absent
      changed_when: false
      check_mode: no

    - name: Registering MariaDB seqno variable
      set_fact:
        seqno: "{{ wsrep_recovery_seqno.stdout_lines[0] }}"
      changed_when: false

    - name: Comparing seqno value on all mariadb hosts
      shell:
        cmd: |
          if [[ ! -z {{ hostvars[inventory_hostname]['seqno'] }} && ! -z {{ hostvars[item]['seqno'] }} &&
          {{ hostvars[inventory_hostname]['seqno'] }} =~ ^-?[0-9]+$ && {{ hostvars[item]['seqno'] }} =~ ^-?[0-9]+$ &&
          {{ hostvars[inventory_hostname]['seqno'] }} -lt {{ hostvars[item]['seqno'] }} ]]; then echo {{ hostvars[item]['seqno'] }}; fi
      with_items: "{{ groups[mariadb_shard_group] }}"
      register: seqno_compare
      args:
        executable: /bin/bash
      changed_when: false

    - name: Writing hostname of host with the largest seqno to temp file
      template:
        src: "hostname.j2"
        dest: "{{ mariadb_recover_tmp_file_path }}"
        mode: "0644"
      delegate_to: localhost
      connection: local
      changed_when: false
      when: seqno_compare.results | map(attribute='stdout') | join('') == ""

    - name: Registering mariadb_recover_inventory_name from temp file
      connection: local
      set_fact:
        mariadb_recover_inventory_name: "{{ lookup('file', mariadb_recover_tmp_file_path) }}"
  when:
    - mariadb_recover_inventory_name is not defined

- name: Store bootstrap and master hostnames into facts
  set_fact:
    bootstrap_host: "{{ mariadb_recover_inventory_name }}"

- name: Set grastate.dat file from MariaDB container in bootstrap host
  become: true
  lineinfile:
    create: yes
    dest: "{{ container_engine_volumes_path }}/mariadb/_data/grastate.dat"
    regexp: 'safe_to_bootstrap:(.*)$'
    line: 'safe_to_bootstrap: 1'
    state: present
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- name: Refresh galera.cnf to set first MariaDB container as primary
  vars:
    service_name: "mariadb"
    service: "{{ mariadb_services[service_name] }}"
    primary_host_on_recovery: "{{ bootstrap_host == inventory_hostname }}"
  merge_configs:
    sources:
      - "{{ role_path }}/templates/galera.cnf.j2"
      - "{{ node_custom_config }}/galera.cnf"
      - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/galera.cnf"
    dest: "{{ node_config_directory }}/{{ service_name }}/galera.cnf"
    mode: "0660"
  become: true
  when:
    - bootstrap_host is defined

- name: Starting first MariaDB container
  become: true
  kolla_container:
    action: "start_container"
    common_options: "{{ docker_common_options }}"
    environment:
      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
      BOOTSTRAP_ARGS: "--wsrep-new-cluster"
    image: "{{ mariadb_service.image }}"
    labels:
      BOOTSTRAP:
    name: "{{ mariadb_service.container_name }}"
    restart_policy: oneshot
    volumes: "{{ mariadb_service.volumes }}"
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- name: Wait for first MariaDB container
  wait_for:
    host: "{{ api_interface_address }}"
    port: "{{ mariadb_port }}"
    connect_timeout: 1
    timeout: 60
    search_regex: "MariaDB"
  register: check_mariadb_port
  until: check_mariadb_port is success
  retries: 10
  delay: 6
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- name: Wait for MariaDB to become operational
  become: true
  kolla_toolbox:
    container_engine: "{{ kolla_container_engine }}"
    module_name: mysql_query
    module_args:
      login_db: "mysql"
      login_host: "{{ api_interface_address }}"
      login_port: "{{ mariadb_port }}"
      login_user: "{{ database_user }}"
      login_password: "{{ database_password }}"
      query: 'SHOW STATUS LIKE "wsrep_evs_state"'
  register: result
  until: result.query_result[0][0]['Value'] == "OPERATIONAL"
  retries: 10
  delay: 6
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- name: Restart slave MariaDB container(s)
  become: true
  kolla_container:
    action: "start_container"
    common_options: "{{ docker_common_options }}"
    name: "{{ item.value.container_name }}"
    image: "{{ item.value.image }}"
    volumes: "{{ item.value.volumes }}"
    dimensions: "{{ item.value.dimensions }}"
    environment: "{{ item.value.environment | default({}) }}"
    healthcheck: "{{ item.value.healthcheck | default(omit) }}"
  with_dict: "{{ mariadb_services | select_services_enabled_and_mapped_to_host }}"
  when:
    - bootstrap_host is defined
    - bootstrap_host != inventory_hostname

- name: Wait for slave MariaDB
  wait_for:
    host: "{{ api_interface_address }}"
    port: "{{ mariadb_port }}"
    connect_timeout: 1
    timeout: 60
    search_regex: "MariaDB"
  register: check_mariadb_port
  until: check_mariadb_port is success
  retries: 10
  delay: 6
  when:
    - bootstrap_host is defined
    - bootstrap_host != inventory_hostname

- name: Unset pc.bootstrap for primary MariaDB galera.cnf for next restart
  vars:
    service_name: "mariadb"
    service: "{{ mariadb_services[service_name] }}"
    primary_host_on_recovery: false
  merge_configs:
    sources:
      - "{{ role_path }}/templates/galera.cnf.j2"
      - "{{ node_custom_config }}/galera.cnf"
      - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/galera.cnf"
    dest: "{{ node_config_directory }}/{{ service_name }}/galera.cnf"
    mode: "0660"
  become: true
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- name: Restart master MariaDB container(s)
  become: true
  kolla_container:
    action: "recreate_or_restart_container"
    common_options: "{{ docker_common_options }}"
    name: "{{ item.value.container_name }}"
    image: "{{ item.value.image }}"
    volumes: "{{ item.value.volumes }}"
    dimensions: "{{ item.value.dimensions }}"
    environment: "{{ item.value.environment | default({}) }}"
    healthcheck: "{{ item.value.healthcheck | default(omit) }}"
  with_dict: "{{ mariadb_services | select_services_enabled_and_mapped_to_host }}"
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- name: Wait for master mariadb
  wait_for:
    host: "{{ api_interface_address }}"
    port: "{{ mariadb_port }}"
    connect_timeout: 1
    timeout: 60
    search_regex: "MariaDB"
  register: check_mariadb_port
  until: check_mariadb_port is success
  retries: 10
  delay: 6
  when:
    - bootstrap_host is defined
    - bootstrap_host == inventory_hostname

- import_tasks: check.yml