Add HAcluster Ansible role

Adds HAcluster Ansible role. This role contains High Availability
clustering solution composed of Corosync, Pacemaker and Pacemaker Remote.

HAcluster is added as a helper role for Masakari which requires it for
its host monitoring, allowing to provide HA to instances on a failed
compute host.

Kolla hacluster images merged in [1].

[1] https://review.opendev.org/#/c/668765/

Change-Id: I91e5c1840ace8f567daf462c4eb3ec1f0c503823
Implements: blueprint ansible-pacemaker-support
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
Co-Authored-By: Mark Goddard <mark@stackhpc.com>

4 files changed, 77 insertions, 12 deletions

diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh
index 8de90907c..5f096c6b0 100755
--- a/tests/setup_gate.sh
+++ b/tests/setup_gate.sh
@@ -78,7 +78,7 @@ function prepare_images {
         GATE_IMAGES+=",^octavia"
     fi
     if [[ $SCENARIO == "masakari" ]]; then
-        GATE_IMAGES+=",^masakari"
+        GATE_IMAGES+=",^masakari-,^hacluster-"
     fi
 
     if [[ $SCENARIO == "swift" ]]; then
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index d746b82e0..f1ea84d3f 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -106,6 +106,7 @@ ironic_dnsmasq_dhcp_range: "10.42.0.2,10.42.0.254"
 
 {% if scenario == "masakari" %}
 enable_masakari: "yes"
+enable_hacluster: "yes"
 {% endif %}
 
 {% if scenario == "cells" %}
diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2
index 27797ae99..e4b05f022 100644
--- a/tests/templates/inventory.j2
+++ b/tests/templates/inventory.j2
@@ -125,6 +125,32 @@ storage
 [elasticsearch:children]
 control
 
+# NOTE(yoctozepto): Until we are able to isolate network namespaces in k-a,
+# we are forced to separate remotes from full members.
+# This is not as bad as it sounds, because it would be enforced in
+# non-containerised environments anyway.
+#[hacluster:children]
+#control
+[hacluster]
+{% for host in hostvars %}
+{% if 'ternary' not in host %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
+{% endif %}
+{% endfor %}
+
+# NOTE(yoctozepto): Until we are able to isolate network namespaces in k-a,
+# we are forced to separate remotes from full members.
+# This is not as bad as it sounds, because it would be enforced in
+# non-containerised environments anyway.
+#[hacluster-remote:children]
+#compute
+[hacluster-remote]
+{% for host in hostvars %}
+{% if 'ternary' in host %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
+{% endif %}
+{% endfor %}
+
 [haproxy:children]
 network
 
diff --git a/tests/test-masakari.sh b/tests/test-masakari.sh
index 28454f508..42a1e26d3 100755
--- a/tests/test-masakari.sh
+++ b/tests/test-masakari.sh
@@ -7,6 +7,52 @@ set -o pipefail
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
+function test_hacluster_logged {
+    local cluster_failure
+    cluster_failure=0
+
+    # NOTE(yoctozepto): repeated -V in commands below is used to get 'debug'
+    # output; the right amount differs between command sets; the next level is
+    # 'trace' which is overly verbose; PCMK_debug=no is used to revert the env
+    # var setting from the container which would cause these commands to log up
+    # to 'trace' (likely a pacemaker bug)
+
+    if ! sudo docker exec hacluster_pacemaker cibadmin -VVVVVV --query --local; then
+        cluster_failure=1
+    fi
+
+    local mon_output
+
+    if ! mon_output=$(sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_mon -VVVVV --one-shot); then
+        cluster_failure=1
+    fi
+
+    if ! sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_verify -VVVVV --live-check; then
+        cluster_failure=1
+    fi
+
+    # NOTE(yoctozepto): crm_mon output should include:
+    #   * Online: [ primary secondary ]
+    #   * RemoteOnline: [ ternary1 ternary2 ]
+
+    if ! echo "$mon_output" | grep 'Online: \[ primary secondary \]'; then
+        echo 'Full members missing' >&2
+        cluster_failure=1
+    fi
+
+    if ! echo "$mon_output" | grep 'RemoteOnline: \[ ternary1 ternary2 \]'; then
+        echo 'Remote members missing' >&2
+        cluster_failure=1
+    fi
+
+    if [[ $cluster_failure -eq 1 ]]; then
+        echo "HAcluster failed"
+        return 1
+    else
+        echo "HAcluster healthy"
+    fi
+}
+
 function test_masakari_logged {
     # Source OpenStack credentials
     . /etc/kolla/admin-openrc.sh
@@ -14,23 +60,14 @@ function test_masakari_logged {
     # Activate virtualenv to access Masakari client
     . ~/openstackclient-venv/bin/activate
 
-    # Get the first Nova compute
-    if ! HYPERVISOR=$(openstack hypervisor list -f value -c 'Hypervisor Hostname' | head -n1); then
-        echo "Unable to get Nova hypervisor list"
-        return 1
-    fi
-
     # Create Masakari segment
     if ! openstack segment create test_segment auto COMPUTE; then
         echo "Unable to create Masakari segment"
         return 1
     fi
 
-    # Add Nova compute to Masakari segment
-    if ! openstack segment host create $HYPERVISOR COMPUTE SSH test_segment; then
-        echo "Unable to add Nova hypervisor to Masakari segment"
-        return 1
-    fi
+    openstack segment host create ternary1 COMPUTE SSH test_segment
+    openstack segment host create ternary2 COMPUTE SSH test_segment
 
     # Delete Masakari segment
     if ! openstack segment delete test_segment; then
@@ -44,6 +81,7 @@ function test_masakari_logged {
 
 function test_masakari {
     echo "Testing Masakari"
+    test_hacluster_logged > /tmp/logs/ansible/test-hacluster 2>&1
     test_masakari_logged > /tmp/logs/ansible/test-masakari 2>&1
     result=$?
     if [[ $result != 0 ]]; then