From af6e1ca4fd3366c1ae936d225b2835e95a4191fc Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 16 Aug 2021 17:03:22 +0100
Subject: Support Ansible max_fail_percentage

This allows us to continue execution until a certain proportion of hosts
to fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kolla_max_fail_percentage, and individual services may define a max
failure percentage via <service>_max_fail_percentage.

Note that all hosts in the inventory must be reachable for fact
gathering, even those not included in a --limit.

Closes-Bug: #1833737
Change-Id: I808474a75c0f0e8b539dc0421374b06cea44be4f
---
 ansible/nova.yml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

(limited to 'ansible/nova.yml')

diff --git a/ansible/nova.yml b/ansible/nova.yml
index 5e12479d7..c41d85407 100644
--- a/ansible/nova.yml
+++ b/ansible/nova.yml
@@ -37,6 +37,10 @@
     - nova-api
     - nova-api-bootstrap
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     # * Create nova API & cell0 DBs & users
     # * API DB schema migrations
@@ -109,6 +113,10 @@
     - nova-api-deploy
     - nova-api-upgrade
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: nova
       when: enable_nova | bool
@@ -130,6 +138,10 @@
     - nova-cell-deploy
     - nova-cell-upgrade
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_cell_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: nova-cell
       when: enable_nova | bool
@@ -147,6 +159,10 @@
     - nova-api
     - nova-refresh-scheduler-cell-cache
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova
@@ -172,6 +188,10 @@
     - nova-api
     - nova-api-reload
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova
@@ -196,6 +216,10 @@
     - nova-cell
     - nova-cell-reload
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_cell_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova-cell
@@ -216,6 +240,10 @@
     - nova-api
     - nova-api-reload
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova
@@ -238,6 +266,8 @@
     - nova-online-data-migrations
     - nova-api-online-data-migrations
   serial: '{{ kolla_serial|default("0") }}'
+  # Fail all hosts if any of these once-per-cell tasks fails.
+  any_errors_fatal: true
   tasks:
     - import_role:
         name: nova
-- 
cgit v1.2.3