From 9f578c85e0b4e498e9bfb7d3795a9db1c545a611 Mon Sep 17 00:00:00 2001 From: Gaëtan Trellu Date: Wed, 10 Jul 2019 11:38:14 -0400 Subject: Add HAcluster Ansible role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds HAcluster Ansible role. This role contains High Availability clustering solution composed of Corosync, Pacemaker and Pacemaker Remote. HAcluster is added as a helper role for Masakari which requires it for its host monitoring, allowing to provide HA to instances on a failed compute host. Kolla hacluster images merged in [1]. [1] https://review.opendev.org/#/c/668765/ Change-Id: I91e5c1840ace8f567daf462c4eb3ec1f0c503823 Implements: blueprint ansible-pacemaker-support Co-Authored-By: Radosław Piliszek Co-Authored-By: Mark Goddard --- tests/setup_gate.sh | 2 +- tests/templates/globals-default.j2 | 1 + tests/templates/inventory.j2 | 26 +++++++++++++++++ tests/test-masakari.sh | 60 +++++++++++++++++++++++++++++++------- 4 files changed, 77 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh index 8de90907c..5f096c6b0 100755 --- a/tests/setup_gate.sh +++ b/tests/setup_gate.sh @@ -78,7 +78,7 @@ function prepare_images { GATE_IMAGES+=",^octavia" fi if [[ $SCENARIO == "masakari" ]]; then - GATE_IMAGES+=",^masakari" + GATE_IMAGES+=",^masakari-,^hacluster-" fi if [[ $SCENARIO == "swift" ]]; then diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index d746b82e0..f1ea84d3f 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -106,6 +106,7 @@ ironic_dnsmasq_dhcp_range: "10.42.0.2,10.42.0.254" {% if scenario == "masakari" %} enable_masakari: "yes" +enable_hacluster: "yes" {% endif %} {% if scenario == "cells" %} diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 27797ae99..e4b05f022 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -125,6 +125,32 @@ storage [elasticsearch:children] control +# NOTE(yoctozepto): Until we are able to isolate network namespaces in k-a, +# we are forced to separate remotes from full members. +# This is not as bad as it sounds, because it would be enforced in +# non-containerised environments anyway. +#[hacluster:children] +#control +[hacluster] +{% for host in hostvars %} +{% if 'ternary' not in host %} +{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }} +{% endif %} +{% endfor %} + +# NOTE(yoctozepto): Until we are able to isolate network namespaces in k-a, +# we are forced to separate remotes from full members. +# This is not as bad as it sounds, because it would be enforced in +# non-containerised environments anyway. +#[hacluster-remote:children] +#compute +[hacluster-remote] +{% for host in hostvars %} +{% if 'ternary' in host %} +{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }} +{% endif %} +{% endfor %} + [haproxy:children] network diff --git a/tests/test-masakari.sh b/tests/test-masakari.sh index 28454f508..42a1e26d3 100755 --- a/tests/test-masakari.sh +++ b/tests/test-masakari.sh @@ -7,6 +7,52 @@ set -o pipefail # Enable unbuffered output for Ansible in Jenkins. export PYTHONUNBUFFERED=1 +function test_hacluster_logged { + local cluster_failure + cluster_failure=0 + + # NOTE(yoctozepto): repeated -V in commands below is used to get 'debug' + # output; the right amount differs between command sets; the next level is + # 'trace' which is overly verbose; PCMK_debug=no is used to revert the env + # var setting from the container which would cause these commands to log up + # to 'trace' (likely a pacemaker bug) + + if ! sudo docker exec hacluster_pacemaker cibadmin -VVVVVV --query --local; then + cluster_failure=1 + fi + + local mon_output + + if ! mon_output=$(sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_mon -VVVVV --one-shot); then + cluster_failure=1 + fi + + if ! sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_verify -VVVVV --live-check; then + cluster_failure=1 + fi + + # NOTE(yoctozepto): crm_mon output should include: + # * Online: [ primary secondary ] + # * RemoteOnline: [ ternary1 ternary2 ] + + if ! echo "$mon_output" | grep 'Online: \[ primary secondary \]'; then + echo 'Full members missing' >&2 + cluster_failure=1 + fi + + if ! echo "$mon_output" | grep 'RemoteOnline: \[ ternary1 ternary2 \]'; then + echo 'Remote members missing' >&2 + cluster_failure=1 + fi + + if [[ $cluster_failure -eq 1 ]]; then + echo "HAcluster failed" + return 1 + else + echo "HAcluster healthy" + fi +} + function test_masakari_logged { # Source OpenStack credentials . /etc/kolla/admin-openrc.sh @@ -14,23 +60,14 @@ function test_masakari_logged { # Activate virtualenv to access Masakari client . ~/openstackclient-venv/bin/activate - # Get the first Nova compute - if ! HYPERVISOR=$(openstack hypervisor list -f value -c 'Hypervisor Hostname' | head -n1); then - echo "Unable to get Nova hypervisor list" - return 1 - fi - # Create Masakari segment if ! openstack segment create test_segment auto COMPUTE; then echo "Unable to create Masakari segment" return 1 fi - # Add Nova compute to Masakari segment - if ! openstack segment host create $HYPERVISOR COMPUTE SSH test_segment; then - echo "Unable to add Nova hypervisor to Masakari segment" - return 1 - fi + openstack segment host create ternary1 COMPUTE SSH test_segment + openstack segment host create ternary2 COMPUTE SSH test_segment # Delete Masakari segment if ! openstack segment delete test_segment; then @@ -44,6 +81,7 @@ function test_masakari_logged { function test_masakari { echo "Testing Masakari" + test_hacluster_logged > /tmp/logs/ansible/test-hacluster 2>&1 test_masakari_logged > /tmp/logs/ansible/test-masakari 2>&1 result=$? if [[ $result != 0 ]]; then -- cgit v1.2.3