blob: 531d0f707d6f9e10bad4fa93c4a162c33750ca6c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
#!/bin/bash
set -o xtrace
set -o errexit
set -o nounset
set -o pipefail
# Enable unbuffered output for Ansible in Jenkins.
export PYTHONUNBUFFERED=1
function mariadb_stop {
echo "Stopping the database cluster"
kolla-ansible stop -i ${RAW_INVENTORY} -vvv --yes-i-really-really-mean-it --tags mariadb --skip-tags common
if [[ $(sudo ${container_engine} ps -q | grep mariadb | wc -l) -ne 0 ]]; then
echo "Failed to stop MariaDB cluster"
return 1
fi
}
function mariadb_recovery {
# Recover the database cluster.
echo "Recovering the database cluster"
kolla-ansible mariadb-recovery -i ${RAW_INVENTORY} -vvv --tags mariadb --skip-tags common
}
function test_recovery {
# Stop all nodes in the cluster, then recover.
mariadb_stop
mariadb_recovery
}
function test_backup {
echo "Performing full backup"
kolla-ansible mariadb-backup -i ${RAW_INVENTORY} -vvv --full
# Sleep for 30 seconds, not because it's absolutely necessary.
# The full backup is already completed at this point, as the
# ansible job is waiting for the completion of the backup script
# in the container on the controller side. It’s more of an
# attempt at a "sort of" simulation of the usual elapsed time
# since the last full backup for the incremental job, as some
# data gets written within those 30 seconds.
echo "Sleeping for 30 seconds"
sleep 30
kolla-ansible mariadb-backup -i ${RAW_INVENTORY} -vvv --incremental
}
function test_backup_with_retries {
# Retry test_backup up to 3 times if it fails
local max_retries=3
local attempt=1
while [[ $attempt -le $max_retries ]]; do
echo "Attempt $attempt of $max_retries for test_backup"
set +o errexit # Temporarily disable errexit for retry logic
test_backup
result=$?
set -o errexit # Re-enable errexit after the attempt
if [[ $result -eq 0 ]]; then
echo "test_backup succeeded on attempt $attempt"
return 0 # Exit the function if test_backup succeeds
else
echo "test_backup failed on attempt $attempt"
fi
if [[ $attempt -lt $max_retries ]]; then
echo "Sleeping for 30 seconds before the next attempt"
sleep 30 # Wait for 30 seconds before retrying
fi
attempt=$((attempt + 1))
done
echo "test_backup failed after $max_retries attempts"
return 1 # Return an error if all attempts fail
}
function test_mariadb_logged {
RAW_INVENTORY=/etc/kolla/inventory
source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
test_backup_with_retries
test_recovery
}
function test_mariadb {
echo "Testing MariaDB"
test_mariadb_logged > /tmp/logs/ansible/test-mariadb 2>&1
result=$?
if [[ $result != 0 ]]; then
echo "Testing MariaDB failed. See ansible/test-mariadb for details"
else
echo "Successfully tested MariaDB. See ansible/test-mariadb for details"
fi
return $result
}
container_engine="${1:-docker}"
test_mariadb
|
