Drop support for building MariaDB clustercheck

MariaDB clustercheck script has not been maintained and it will not work
with MariaDB 11.4 (the next LTS version).
This is because clustercheck.sh uses ``mysql*`` commands but from
MariaDB 11.4, those commands are not supported.

The health check of MariaDB server containers will be done by new script
which comes with change [1].

Without MariaDB clustercheck, MariaDB server becomes only container image in
the family, so mariadb-base and mariadb-server dockerfiles are merged.
To avoid breaking changes, the name 'mariadb-server' is retained.
Follow-up to rename references in Kolla and K-A from 'mariadb-server' to
'mariadb' would be nice.

[1] https://review.opendev.org/c/openstack/kolla/+/962657

Change-Id: I01278f76ee82b018a854b5862f3caaa8c78a9a30
Signed-off-by: Seunghun Lee <seunghun@stackhpc.com>

7 files changed, 465 insertions, 0 deletions

diff --git a/docker/mariadb-server/Dockerfile.j2 b/docker/mariadb-server/Dockerfile.j2
new file mode 100644
index 000000000..ddb045fbb
--- /dev/null
+++ b/docker/mariadb-server/Dockerfile.j2
@@ -0,0 +1,96 @@
+FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
+{% block labels %}
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+{% endblock %}
+
+{% block mariadb_header %}{% endblock %}
+
+{% import "macros.j2" as macros with context %}
+
+{{ macros.configure_user(name='mysql') }}
+
+{# NOTE(mnasiadka): Use AppStream version of MariaDB for now in CentOS #}
+
+{% if base_package_type == 'rpm' %}
+
+{# NOTE(mgoddard): EPEL required for pv package #}
+{{ macros.enable_extra_repos(['epel']) }}
+
+    {% set mariadb_packages = [
+        'expect',
+        'mariadb',
+        'mariadb-backup',
+        'mariadb-server',
+        'mariadb-server-galera',
+        'pv',
+        'rsync',
+        'tar'
+    ] %}
+
+{# NOTE(yoctozepto): this is to ensure Kolla Ansible can configure MariaDB to find it #}
+RUN ln -s /usr/lib64/galera-4 /usr/lib64/galera
+
+{% elif base_package_type == 'deb' %}
+{{ macros.enable_extra_repos(['mariadb']) }}
+    {% set mariadb_packages = [
+        'expect',
+        'mariadb-backup',
+        'mariadb-client',
+        'mariadb-server'
+    ] %}
+{% endif %}
+
+{{ macros.install_packages(mariadb_packages | customizable("packages")) }}
+
+{% block mariadb_healthcheck %}
+ENV MARIADB_VERSION=10.11
+ADD plugins-archive /
+RUN install -m 755 /plugins/mariadb-server-plugin-mariadb-docker-archive*/$MARIADB_VERSION/healthcheck.sh /usr/bin/healthcheck.sh
+{% endblock %}
+
+COPY mariadb_sudoers /etc/sudoers.d/kolla_mariadb_sudoers
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+COPY security_reset.expect /usr/local/bin/kolla_security_reset
+COPY backup.sh /usr/local/bin/kolla_mariadb_backup.sh
+COPY backup_replica.sh /usr/local/bin/kolla_mariadb_backup_replica.sh
+
+RUN chmod 644 /usr/local/bin/kolla_extend_start \
+    && chmod 755 /usr/local/bin/kolla_security_reset \
+                 /usr/local/bin/kolla_mariadb_backup.sh \
+                 /usr/local/bin/kolla_mariadb_backup_replica.sh \
+    && chmod 750 /etc/sudoers.d \
+    && chmod 440 /etc/sudoers.d/kolla_mariadb_sudoers \
+    && rm -rf /var/lib/mysql/*
+
+{% if base_package_type == 'deb' %}
+RUN mkdir -p /var/run/mysqld && chown mysql /var/run/mysqld && chmod 755 /var/run/mysqld
+{% endif %}
+
+{% if docker_healthchecks %}
+{% block healthcheck_installation %}
+
+COPY healthcheck_mariadb /usr/local/bin/healthcheck_mariadb
+RUN chmod 755 /usr/local/bin/healthcheck_mariadb
+
+{% endblock %}
+{% endif %}
+
+{% if use_dumb_init %}
+{% block mariadb_entrypoint %}
+# NOTE(mgoddard): Override the dumb-init arguments to avoid passing
+# --single-child. This does not play well with mysqld_safe, which ignores
+# SIGTERM, meaning Docker needs to forcibly kill the container to stop it.
+# Without --single-child, the TERM signal is sent to all subprocesses,
+# including mysqld.
+
+ENTRYPOINT ["dumb-init", "--"]
+CMD ["kolla_start"]
+{% endblock %}
+{% endif %}
+
+{{ macros.kolla_patch_sources() }}
+
+{% block mariadb_footer %}{% endblock %}
+{% block footer %}{% endblock %}
+
+USER mysql
diff --git a/docker/mariadb-server/backup.sh b/docker/mariadb-server/backup.sh
new file mode 100644
index 000000000..be63cd938
--- /dev/null
+++ b/docker/mariadb-server/backup.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+# Execute a full backup
+backup_full() {
+    echo "Taking a full backup"
+    LAST_FULL_DATE=$(date +%d-%m-%Y-%s)
+    BACKUP_FILE="mysqlbackup-${LAST_FULL_DATE}.qp.xbc.xbs.gz"
+    BACKUP_PATH="$BACKUP_DIR/full-${LAST_FULL_DATE}"
+    mkdir -p "$BACKUP_PATH"
+
+    mariabackup \
+        --defaults-file=/etc/mysql/my.cnf \
+        --backup \
+        --stream=xbstream \
+        --history=$LAST_FULL_DATE | gzip > \
+        "$BACKUP_PATH/$BACKUP_FILE"
+
+    echo "$BACKUP_PATH/$BACKUP_FILE" > "$BACKUP_DIR/last_full_file"
+}
+
+# Execute an incremental backup
+backup_incremental() {
+    echo "Taking an incremental backup"
+
+    if [ ! -r "$BACKUP_DIR/last_full_file" ]; then
+        echo "Error: No full backup file found."
+        exit 1
+    fi
+
+    FULL_BACKUP_FILE=$(cat "$BACKUP_DIR/last_full_file")
+    LAST_FULL_DATE=$(basename "$(dirname "$FULL_BACKUP_FILE")" | sed 's/^full-//')
+    NOW=$(date +%H-%M-%S-%d-%m-%Y)
+    INCR_DIR="$BACKUP_DIR/incr-${NOW}-since-${LAST_FULL_DATE}"
+    mkdir -p "$INCR_DIR"
+
+    TMP_BASEDIR=$(mktemp -d)
+    echo "Decompressing full backup to temp dir: $TMP_BASEDIR"
+    gunzip -c "$FULL_BACKUP_FILE" | mbstream -x -C "$TMP_BASEDIR"
+
+    mariabackup \
+        --defaults-file=/etc/mysql/my.cnf \
+        --backup \
+        --stream=xbstream \
+        --incremental-basedir="$TMP_BASEDIR" \
+        --history="incr-${NOW}" | gzip > \
+        "$INCR_DIR/incremental-${NOW}-mysqlbackup-${LAST_FULL_DATE}.qp.xbc.xbs.gz"
+
+    rm -rf "$TMP_BASEDIR"
+}
+
+BACKUP_DIR=/backup/
+cd "$BACKUP_DIR"
+
+if [ -n "${BACKUP_TYPE:-}" ]; then
+    case "$BACKUP_TYPE" in
+        "full")
+            backup_full
+            ;;
+        "incremental")
+            backup_incremental
+            ;;
+        *)
+            echo "Only full or incremental options are supported."
+            exit 1
+            ;;
+    esac
+else
+    echo "You need to specify either full or incremental backup options."
+    exit 1
+fi
diff --git a/docker/mariadb-server/backup_replica.sh b/docker/mariadb-server/backup_replica.sh
new file mode 100644
index 000000000..ebdcbf6b0
--- /dev/null
+++ b/docker/mariadb-server/backup_replica.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+BACKUP_DIR=/backup/
+DEFAULT_MY_CNF="/etc/mysql/my.cnf"
+REPLICA_MY_CNF="$(mktemp)"
+RETRY_INTERVAL=5  # Interval between retries (in seconds)
+MAX_RETRIES=12    # Max retries (12 retries * 5 seconds = 60 seconds)
+
+# Cleanup function to remove the REPLICA_MY_CNF file
+cleanup() {
+    rm -f "${REPLICA_MY_CNF}"
+}
+
+# Set trap to ensure cleanup occurs on exit or error
+trap cleanup EXIT
+
+cd "${BACKUP_DIR}"
+
+# Execute a full backup
+backup_full() {
+    echo "Taking a full backup"
+    LAST_FULL_DATE=$(date +%d-%m-%Y-%s)
+    BACKUP_FILE="backup-full-${LAST_FULL_DATE}.mbs.gz"
+    BACKUP_PATH="${BACKUP_DIR}/full-${LAST_FULL_DATE}"
+    mkdir -p "${BACKUP_PATH}"
+
+    mariabackup \
+        --defaults-file="${REPLICA_MY_CNF}" \
+        --backup \
+        --stream=mbstream \
+        --history="${LAST_FULL_DATE}" \
+        | gzip > "${BACKUP_PATH}/${BACKUP_FILE}"
+
+    echo "${BACKUP_PATH}/${BACKUP_FILE}" > "${BACKUP_DIR}/last_full_file"
+}
+
+# Execute an incremental backup
+backup_incremental() {
+    if [ ! -r "${BACKUP_DIR}/last_full_file" ]; then
+        echo "Error: No full backup file found."
+        exit 1
+    fi
+
+    FULL_BACKUP_FILE=$(cat "${BACKUP_DIR}/last_full_file")
+    LAST_FULL_DATE=$(basename "$(dirname "${FULL_BACKUP_FILE}")" | sed 's/^full-//')
+    NOW=$(date +%H-%M-%S-%d-%m-%Y)
+    INCR_DIR="${BACKUP_DIR}/incr-${NOW}-since-${LAST_FULL_DATE}"
+    mkdir -p "${INCR_DIR}"
+
+    # Temp dir for full base restore
+    TMP_BASEDIR=$(mktemp -d)
+
+    echo "Decompressing full backup to temp dir: ${TMP_BASEDIR}"
+    gunzip -c "${FULL_BACKUP_FILE}" | mbstream -x -C "${TMP_BASEDIR}"
+
+    mariabackup \
+        --defaults-file="${REPLICA_MY_CNF}" \
+        --backup \
+        --stream=mbstream \
+        --incremental-basedir="${TMP_BASEDIR}" \
+        --history="incr-${NOW}" \
+        | gzip > "${INCR_DIR}/backup-incremental-${NOW}.mbs.gz"
+
+    rm -rf "${TMP_BASEDIR}"
+}
+
+# Retry logic for database queries
+retry_mysql_query() {
+    local query="$1"
+    local result=""
+    local attempt=1
+
+    while [ ${attempt} -le ${MAX_RETRIES} ]; do
+        result=$(mysql -h "${HOST}" -u "${USER}" -p"${PASS}" -s -N -e "${query}" 2>/dev/null || true)
+        if [ -n "${result}" ]; then
+            echo "${result}"
+            return 0
+        fi
+        echo "Attempt ${attempt}/${MAX_RETRIES} failed. Retrying in ${RETRY_INTERVAL} seconds..."
+        sleep "${RETRY_INTERVAL}"
+        attempt=$((attempt + 1))
+    done
+
+    echo "Error: Failed to execute the query after ${MAX_RETRIES} attempts."
+    return 1
+}
+
+get_and_set_replica_server() {
+    HOST="$(grep '^host' "${DEFAULT_MY_CNF}" | awk -F '=' '{print $2}' | xargs)"
+    USER="$(grep '^user' "${DEFAULT_MY_CNF}" | awk -F '=' '{print $2}' | xargs)"
+    PASS="$(grep '^password' "${DEFAULT_MY_CNF}" | awk -F '=' '{print $2}' | xargs)"
+
+    ALL_HOSTS_SELECT="SELECT REGEXP_REPLACE(VARIABLE_VALUE, ':[0-9]*','') FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_incoming_addresses';"
+    ALL_HOSTS=$(retry_mysql_query "${ALL_HOSTS_SELECT}")
+    if [ -z "${ALL_HOSTS}" ]; then
+        echo "Backup failed due to inability to fetch a list of servers."
+        exit 1
+    fi
+
+    ACTIVE_HOST_SELECT='SELECT @@hostname;'
+    ACTIVE_HOST=$(retry_mysql_query "${ACTIVE_HOST_SELECT}" | xargs getent hosts | awk '{print $1}')
+    if [ -z "${ACTIVE_HOST}" ]; then
+        echo "Backup failed due to inability to fetch active host."
+        exit 1
+    fi
+
+    # Multinode
+    if echo "${ALL_HOSTS}" | grep -q ','; then
+        for server in $(echo "${ALL_HOSTS}" | tr ',' '\n'); do
+            if [[ "${server}" != "${ACTIVE_HOST}" ]]; then
+                REPLICA_HOST="${server}"
+                break
+            fi
+        done
+    # Single node
+    else
+        REPLICA_HOST="${ALL_HOSTS}"
+    fi
+    if [ -z "${REPLICA_HOST}" ]; then
+        echo "Backup failed due to inability to determine replica host."
+        exit 1
+    fi
+
+    cp "${DEFAULT_MY_CNF}" "${REPLICA_MY_CNF}"
+    sed -i "s/${HOST}/${REPLICA_HOST}/g" "${REPLICA_MY_CNF}"
+}
+
+if [ -n "${BACKUP_TYPE}" ]; then
+    get_and_set_replica_server
+    case "${BACKUP_TYPE}" in
+        "full")
+            backup_full
+            ;;
+        "incremental")
+            backup_incremental
+            ;;
+        *)
+            echo "Only full or incremental options are supported."
+            exit 1
+            ;;
+    esac
+else
+    echo "You need to specify either full or incremental backup options."
+    exit 1
+fi
diff --git a/docker/mariadb-server/extend_start.sh b/docker/mariadb-server/extend_start.sh
new file mode 100644
index 000000000..53a54557d
--- /dev/null
+++ b/docker/mariadb-server/extend_start.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+: ${MARIADB_LOG_DIR:=/var/log/kolla/mariadb}
+
+function bootstrap_db {
+    mysqld_safe --wsrep-new-cluster --skip-networking --wsrep-on=OFF --pid-file=/var/lib/mysql/mariadb.pid &
+    # Wait for the mariadb server to be "Ready" before starting the security reset with a max timeout
+    # NOTE(huikang): the location of mysql's socket file varies depending on the OS distributions.
+    # Querying the cluster status has to be executed after the existence of mysql.sock and mariadb.pid.
+    TIMEOUT=${DB_MAX_TIMEOUT:-60}
+    while [[ ! -S /var/lib/mysql/mysql.sock ]] && \
+          [[ ! -S /var/run/mysqld/mysqld.sock ]] || \
+          [[ ! -f /var/lib/mysql/mariadb.pid ]]; do
+        if [[ ${TIMEOUT} -gt 0 ]]; then
+            let TIMEOUT-=1
+            sleep 1
+        else
+            exit 1
+        fi
+    done
+
+    sudo -E kolla_security_reset
+
+    set +o xtrace
+    mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
+    mysql -u root --password="${DB_ROOT_PASSWORD}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${DB_ROOT_PASSWORD}' WITH GRANT OPTION;"
+    mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown
+    set -o xtrace
+}
+
+# Create log directory, with appropriate permissions
+if [[ ! -d "${MARIADB_LOG_DIR}" ]]; then
+    mkdir -p ${MARIADB_LOG_DIR}
+fi
+if [[ $(stat -c %a ${MARIADB_LOG_DIR}) != "755" ]]; then
+    chmod 755 ${MARIADB_LOG_DIR}
+fi
+
+# This catches all cases of the BOOTSTRAP variable being set, including empty
+if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
+    mysql_install_db 2>&1 | tee -a ${MARIADB_LOG_DIR}/mariadb-bootstrap.log
+    bootstrap_db 2>&1 | tee -a ${MARIADB_LOG_DIR}/mariadb-bootstrap.log
+    exit 0
+fi
+
+# This catches all cases of the KOLLA_UPGRADE variable being set, including empty
+if [[ "${!KOLLA_UPGRADE[@]}" ]]; then
+    # The mysql_upgrade command treats any directories under /var/lib/mysql as
+    # databases. Somehow we can end up with a .pki directory, which causes the
+    # command to fail with this error:
+    # Incorrect database name '#mysql50#.pki' when selecting the database
+    # There doesn't seem to be anything in the directory, so remove it.
+    rm -rf /var/lib/mysql/.pki
+
+    mysql_upgrade --host=${DB_HOST} --port=${DB_PORT} --user=root --password="${DB_ROOT_PASSWORD}" 2>&1 | tee -a ${MARIADB_LOG_DIR}/mariadb-upgrade.log
+    exit 0
+fi
+
+if [[ "${!BOOTSTRAP_ARGS[@]}" ]]; then
+    ARGS="${BOOTSTRAP_ARGS}"
+fi
diff --git a/docker/mariadb-server/healthcheck_mariadb b/docker/mariadb-server/healthcheck_mariadb
new file mode 100755
index 000000000..0d9f59a95
--- /dev/null
+++ b/docker/mariadb-server/healthcheck_mariadb
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+MYSQL_USERNAME="${MYSQL_USERNAME:=-haproxy}"
+MYSQL_TIMEOUT=10
+
+MYSQL_CMDLINE="mysql -sN --connect-timeout=${MYSQL_TIMEOUT} -u ${MYSQL_USERNAME}"
+
+WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state_comment';" | awk '{print $2}')
+
+if [[ "${WSREP_STATUS}" == "Synced" ]]
+then
+    echo "MariaDB Galera Cluster Node is synced."
+    exit 0
+else
+    echo "MariaDB Galera Cluster Node is NOT synced"
+    exit 1
+fi
diff --git a/docker/mariadb-server/mariadb_sudoers b/docker/mariadb-server/mariadb_sudoers
new file mode 100644
index 000000000..150534e16
--- /dev/null
+++ b/docker/mariadb-server/mariadb_sudoers
@@ -0,0 +1 @@
+%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_security_reset
diff --git a/docker/mariadb-server/security_reset.expect b/docker/mariadb-server/security_reset.expect
new file mode 100644
index 000000000..06f1d5c8e
--- /dev/null
+++ b/docker/mariadb-server/security_reset.expect
@@ -0,0 +1,67 @@
+#!/usr/bin/expect -f
+
+if [catch {set timeout $env(DB_MAX_TIMEOUT)}] {set timeout 10}
+spawn mariadb-secure-installation
+expect {
+  timeout { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 }
+  "Enter current password for root (enter for none):"
+}
+send "\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Switch to unix_socket authentication [Y/n] ' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Switch to unix_socket authentication [Y/n]' prompt\n"; exit 1 }
+  "Switch to unix_socket authentication \\\[Y/n\\\] "
+}
+send "n\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Change the root password? [Y/n]' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Change the root password? [Y/n]' prompt\n"; exit 1 }
+  "Change the root password? \\\[Y/n\\\] "
+}
+send "y\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 }
+  "New password:"
+}
+send "$env(DB_ROOT_PASSWORD)\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 }
+  "Re-enter new password:"
+}
+send "$env(DB_ROOT_PASSWORD)\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 }
+  "Remove anonymous users?"
+}
+send "y\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 }
+  "Disallow root login remotely?"
+}
+send "n\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 }
+  "Remove test database and access to it?"
+}
+send "y\r"
+
+expect {
+  timeout { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  eof { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 }
+  "Reload privilege tables now?"
+}
+send "y\r"
+expect eof