From 9f578c85e0b4e498e9bfb7d3795a9db1c545a611 Mon Sep 17 00:00:00 2001 From: Gaëtan Trellu Date: Wed, 10 Jul 2019 11:38:14 -0400 Subject: Add HAcluster Ansible role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds HAcluster Ansible role. This role contains High Availability clustering solution composed of Corosync, Pacemaker and Pacemaker Remote. HAcluster is added as a helper role for Masakari which requires it for its host monitoring, allowing to provide HA to instances on a failed compute host. Kolla hacluster images merged in [1]. [1] https://review.opendev.org/#/c/668765/ Change-Id: I91e5c1840ace8f567daf462c4eb3ec1f0c503823 Implements: blueprint ansible-pacemaker-support Co-Authored-By: Radosław Piliszek Co-Authored-By: Mark Goddard --- tests/test-masakari.sh | 60 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 11 deletions(-) (limited to 'tests/test-masakari.sh') diff --git a/tests/test-masakari.sh b/tests/test-masakari.sh index 28454f508..42a1e26d3 100755 --- a/tests/test-masakari.sh +++ b/tests/test-masakari.sh @@ -7,6 +7,52 @@ set -o pipefail # Enable unbuffered output for Ansible in Jenkins. export PYTHONUNBUFFERED=1 +function test_hacluster_logged { + local cluster_failure + cluster_failure=0 + + # NOTE(yoctozepto): repeated -V in commands below is used to get 'debug' + # output; the right amount differs between command sets; the next level is + # 'trace' which is overly verbose; PCMK_debug=no is used to revert the env + # var setting from the container which would cause these commands to log up + # to 'trace' (likely a pacemaker bug) + + if ! sudo docker exec hacluster_pacemaker cibadmin -VVVVVV --query --local; then + cluster_failure=1 + fi + + local mon_output + + if ! mon_output=$(sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_mon -VVVVV --one-shot); then + cluster_failure=1 + fi + + if ! sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_verify -VVVVV --live-check; then + cluster_failure=1 + fi + + # NOTE(yoctozepto): crm_mon output should include: + # * Online: [ primary secondary ] + # * RemoteOnline: [ ternary1 ternary2 ] + + if ! echo "$mon_output" | grep 'Online: \[ primary secondary \]'; then + echo 'Full members missing' >&2 + cluster_failure=1 + fi + + if ! echo "$mon_output" | grep 'RemoteOnline: \[ ternary1 ternary2 \]'; then + echo 'Remote members missing' >&2 + cluster_failure=1 + fi + + if [[ $cluster_failure -eq 1 ]]; then + echo "HAcluster failed" + return 1 + else + echo "HAcluster healthy" + fi +} + function test_masakari_logged { # Source OpenStack credentials . /etc/kolla/admin-openrc.sh @@ -14,23 +60,14 @@ function test_masakari_logged { # Activate virtualenv to access Masakari client . ~/openstackclient-venv/bin/activate - # Get the first Nova compute - if ! HYPERVISOR=$(openstack hypervisor list -f value -c 'Hypervisor Hostname' | head -n1); then - echo "Unable to get Nova hypervisor list" - return 1 - fi - # Create Masakari segment if ! openstack segment create test_segment auto COMPUTE; then echo "Unable to create Masakari segment" return 1 fi - # Add Nova compute to Masakari segment - if ! openstack segment host create $HYPERVISOR COMPUTE SSH test_segment; then - echo "Unable to add Nova hypervisor to Masakari segment" - return 1 - fi + openstack segment host create ternary1 COMPUTE SSH test_segment + openstack segment host create ternary2 COMPUTE SSH test_segment # Delete Masakari segment if ! openstack segment delete test_segment; then @@ -44,6 +81,7 @@ function test_masakari_logged { function test_masakari { echo "Testing Masakari" + test_hacluster_logged > /tmp/logs/ansible/test-hacluster 2>&1 test_masakari_logged > /tmp/logs/ansible/test-masakari 2>&1 result=$? if [[ $result != 0 ]]; then -- cgit v1.2.3