Add HAcluster Ansible role

Adds HAcluster Ansible role. This role contains High Availability
clustering solution composed of Corosync, Pacemaker and Pacemaker Remote.

HAcluster is added as a helper role for Masakari which requires it for
its host monitoring, allowing to provide HA to instances on a failed
compute host.

Kolla hacluster images merged in [1].

[1] https://review.opendev.org/#/c/668765/

Change-Id: I91e5c1840ace8f567daf462c4eb3ec1f0c503823
Implements: blueprint ansible-pacemaker-support
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
Co-Authored-By: Mark Goddard <mark@stackhpc.com>

1 files changed, 49 insertions, 11 deletions

diff --git a/tests/test-masakari.sh b/tests/test-masakari.sh
index 28454f508..42a1e26d3 100755
--- a/tests/test-masakari.sh
+++ b/tests/test-masakari.sh
@@ -7,6 +7,52 @@ set -o pipefail
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
+function test_hacluster_logged {
+    local cluster_failure
+    cluster_failure=0
+
+    # NOTE(yoctozepto): repeated -V in commands below is used to get 'debug'
+    # output; the right amount differs between command sets; the next level is
+    # 'trace' which is overly verbose; PCMK_debug=no is used to revert the env
+    # var setting from the container which would cause these commands to log up
+    # to 'trace' (likely a pacemaker bug)
+
+    if ! sudo docker exec hacluster_pacemaker cibadmin -VVVVVV --query --local; then
+        cluster_failure=1
+    fi
+
+    local mon_output
+
+    if ! mon_output=$(sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_mon -VVVVV --one-shot); then
+        cluster_failure=1
+    fi
+
+    if ! sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_verify -VVVVV --live-check; then
+        cluster_failure=1
+    fi
+
+    # NOTE(yoctozepto): crm_mon output should include:
+    #   * Online: [ primary secondary ]
+    #   * RemoteOnline: [ ternary1 ternary2 ]
+
+    if ! echo "$mon_output" | grep 'Online: \[ primary secondary \]'; then
+        echo 'Full members missing' >&2
+        cluster_failure=1
+    fi
+
+    if ! echo "$mon_output" | grep 'RemoteOnline: \[ ternary1 ternary2 \]'; then
+        echo 'Remote members missing' >&2
+        cluster_failure=1
+    fi
+
+    if [[ $cluster_failure -eq 1 ]]; then
+        echo "HAcluster failed"
+        return 1
+    else
+        echo "HAcluster healthy"
+    fi
+}
+
 function test_masakari_logged {
     # Source OpenStack credentials
     . /etc/kolla/admin-openrc.sh
@@ -14,23 +60,14 @@ function test_masakari_logged {
     # Activate virtualenv to access Masakari client
     . ~/openstackclient-venv/bin/activate
 
-    # Get the first Nova compute
-    if ! HYPERVISOR=$(openstack hypervisor list -f value -c 'Hypervisor Hostname' | head -n1); then
-        echo "Unable to get Nova hypervisor list"
-        return 1
-    fi
-
     # Create Masakari segment
     if ! openstack segment create test_segment auto COMPUTE; then
         echo "Unable to create Masakari segment"
         return 1
     fi
 
-    # Add Nova compute to Masakari segment
-    if ! openstack segment host create $HYPERVISOR COMPUTE SSH test_segment; then
-        echo "Unable to add Nova hypervisor to Masakari segment"
-        return 1
-    fi
+    openstack segment host create ternary1 COMPUTE SSH test_segment
+    openstack segment host create ternary2 COMPUTE SSH test_segment
 
     # Delete Masakari segment
     if ! openstack segment delete test_segment; then
@@ -44,6 +81,7 @@ function test_masakari_logged {
 
 function test_masakari {
     echo "Testing Masakari"
+    test_hacluster_logged > /tmp/logs/ansible/test-hacluster 2>&1
     test_masakari_logged > /tmp/logs/ansible/test-masakari 2>&1
     result=$?
     if [[ $result != 0 ]]; then