Merge "Rabbitmq: enable quorum for transient queues"

30 files changed, 142 insertions, 1 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 8fb995656..859fbeb2e 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -278,6 +278,11 @@ om_enable_rabbitmq_high_availability: false
 # Only enable quorum queues if you disable om_enable_rabbitmq_high_availability
 om_enable_rabbitmq_quorum_queues: true
 om_enable_queue_manager: true
+# Enabling om_enable_rabbitmq_transient_quorum_queue will have no effect
+# without also enabling om_enable_rabbitmq_quorum_queues.
+# om_enable_rabbitmq_transient_quorum_queue should only be used in conunction
+# with om_enable_queue_manager.
+om_enable_rabbitmq_transient_quorum_queue: "{{ om_enable_rabbitmq_quorum_queues | bool and om_enable_queue_manager | bool }}"
 
 ####################
 # Networking options
diff --git a/ansible/roles/aodh/templates/aodh.conf.j2 b/ansible/roles/aodh/templates/aodh.conf.j2
index 9c7ca11d9..f94a49196 100644
--- a/ansible/roles/aodh/templates/aodh.conf.j2
+++ b/ansible/roles/aodh/templates/aodh.conf.j2
@@ -77,6 +77,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/barbican/templates/barbican.conf.j2 b/ansible/roles/barbican/templates/barbican.conf.j2
index 00de9baa9..f5827446b 100644
--- a/ansible/roles/barbican/templates/barbican.conf.j2
+++ b/ansible/roles/barbican/templates/barbican.conf.j2
@@ -97,6 +97,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/blazar/templates/blazar.conf.j2 b/ansible/roles/blazar/templates/blazar.conf.j2
index e75c91868..7ec239440 100644
--- a/ansible/roles/blazar/templates/blazar.conf.j2
+++ b/ansible/roles/blazar/templates/blazar.conf.j2
@@ -68,6 +68,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if blazar_policy_file is defined %}
diff --git a/ansible/roles/ceilometer/templates/ceilometer.conf.j2 b/ansible/roles/ceilometer/templates/ceilometer.conf.j2
index c39e73ddb..a09d5125c 100644
--- a/ansible/roles/ceilometer/templates/ceilometer.conf.j2
+++ b/ansible/roles/ceilometer/templates/ceilometer.conf.j2
@@ -49,6 +49,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if ceilometer_policy_file is defined %}
diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2
index 969bda689..62664b623 100644
--- a/ansible/roles/cinder/templates/cinder.conf.j2
+++ b/ansible/roles/cinder/templates/cinder.conf.j2
@@ -87,6 +87,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
index 0526e1133..409ce44c8 100644
--- a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
+++ b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
@@ -61,6 +61,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [collect]
diff --git a/ansible/roles/cyborg/templates/cyborg.conf.j2 b/ansible/roles/cyborg/templates/cyborg.conf.j2
index 58b1169bf..8b74a7622 100644
--- a/ansible/roles/cyborg/templates/cyborg.conf.j2
+++ b/ansible/roles/cyborg/templates/cyborg.conf.j2
@@ -83,6 +83,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/designate/templates/designate.conf.j2 b/ansible/roles/designate/templates/designate.conf.j2
index 2b61fce59..964ea2e5b 100644
--- a/ansible/roles/designate/templates/designate.conf.j2
+++ b/ansible/roles/designate/templates/designate.conf.j2
@@ -106,6 +106,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/glance/templates/glance-api.conf.j2 b/ansible/roles/glance/templates/glance-api.conf.j2
index dcbcd6972..66d3533ff 100644
--- a/ansible/roles/glance/templates/glance-api.conf.j2
+++ b/ansible/roles/glance/templates/glance-api.conf.j2
@@ -126,6 +126,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if glance_policy_file is defined %}
diff --git a/ansible/roles/heat/templates/heat.conf.j2 b/ansible/roles/heat/templates/heat.conf.j2
index 2121996b2..fcae67459 100644
--- a/ansible/roles/heat/templates/heat.conf.j2
+++ b/ansible/roles/heat/templates/heat.conf.j2
@@ -88,6 +88,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if heat_policy_file is defined %}
diff --git a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 b/ansible/roles/ironic/templates/ironic-inspector.conf.j2
index 5cfb96bd8..f6c1f5622 100644
--- a/ansible/roles/ironic/templates/ironic-inspector.conf.j2
+++ b/ansible/roles/ironic/templates/ironic-inspector.conf.j2
@@ -26,6 +26,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [ironic]
diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2
index 92e4e485e..9b3bcffa4 100644
--- a/ansible/roles/ironic/templates/ironic.conf.j2
+++ b/ansible/roles/ironic/templates/ironic.conf.j2
@@ -57,6 +57,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if ironic_policy_file is defined %}
diff --git a/ansible/roles/keystone/templates/keystone.conf.j2 b/ansible/roles/keystone/templates/keystone.conf.j2
index cf44e0991..07bb7f04d 100644
--- a/ansible/roles/keystone/templates/keystone.conf.j2
+++ b/ansible/roles/keystone/templates/keystone.conf.j2
@@ -76,6 +76,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if enable_osprofiler | bool %}
diff --git a/ansible/roles/magnum/templates/magnum.conf.j2 b/ansible/roles/magnum/templates/magnum.conf.j2
index ce97de7ab..27a76d48d 100644
--- a/ansible/roles/magnum/templates/magnum.conf.j2
+++ b/ansible/roles/magnum/templates/magnum.conf.j2
@@ -140,6 +140,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if magnum_policy_file is defined %}
diff --git a/ansible/roles/manila/templates/manila.conf.j2 b/ansible/roles/manila/templates/manila.conf.j2
index 2383380e3..45d3a69d4 100644
--- a/ansible/roles/manila/templates/manila.conf.j2
+++ b/ansible/roles/manila/templates/manila.conf.j2
@@ -70,6 +70,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/masakari/templates/masakari.conf.j2 b/ansible/roles/masakari/templates/masakari.conf.j2
index dff6040b4..3c0547221 100644
--- a/ansible/roles/masakari/templates/masakari.conf.j2
+++ b/ansible/roles/masakari/templates/masakari.conf.j2
@@ -67,6 +67,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/mistral/templates/mistral.conf.j2 b/ansible/roles/mistral/templates/mistral.conf.j2
index c7f03475f..a983fcd51 100644
--- a/ansible/roles/mistral/templates/mistral.conf.j2
+++ b/ansible/roles/mistral/templates/mistral.conf.j2
@@ -86,6 +86,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if mistral_policy_file is defined %}
diff --git a/ansible/roles/neutron/templates/neutron.conf.j2 b/ansible/roles/neutron/templates/neutron.conf.j2
index 3b7e4063b..178312a68 100644
--- a/ansible/roles/neutron/templates/neutron.conf.j2
+++ b/ansible/roles/neutron/templates/neutron.conf.j2
@@ -145,6 +145,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if neutron_policy_file is defined %}
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index a52132e31..9db0bf050 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -205,6 +205,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}
diff --git a/ansible/roles/nova/templates/nova.conf.j2 b/ansible/roles/nova/templates/nova.conf.j2
index 0b7680f81..6be659dd0 100644
--- a/ansible/roles/nova/templates/nova.conf.j2
+++ b/ansible/roles/nova/templates/nova.conf.j2
@@ -155,6 +155,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}
diff --git a/ansible/roles/octavia/templates/octavia.conf.j2 b/ansible/roles/octavia/templates/octavia.conf.j2
index ce28a9dda..eed5bf954 100644
--- a/ansible/roles/octavia/templates/octavia.conf.j2
+++ b/ansible/roles/octavia/templates/octavia.conf.j2
@@ -140,6 +140,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if octavia_policy_file is defined %}
diff --git a/ansible/roles/rabbitmq/tasks/precheck.yml b/ansible/roles/rabbitmq/tasks/precheck.yml
index 3df9548b8..a56b0ebe4 100644
--- a/ansible/roles/rabbitmq/tasks/precheck.yml
+++ b/ansible/roles/rabbitmq/tasks/precheck.yml
@@ -149,9 +149,24 @@
       loop: "{{ (rabbitmq_queues.stdout | from_json) if rabbitmq_queues is not skipped else [] }}"
       loop_control:
         label: "{{ item.name }}"
-      # TODO(mattcress): remove skipping reply and fanout once https://review.opendev.org/c/openstack/oslo.messaging/+/888479 is merged.
       when: not (item.name is search('reply_') or item.name is search('_fanout'))
 
+    - name: Check if RabbitMQ quorum queues for transient queues need to be configured
+      assert:
+        that: "{{ item.type == 'quorum' }}"
+        fail_msg: >
+          om_enable_rabbitmq_transient_quorum_queue is True but {{ item.name }} is a non-quorum queue.
+          Currently the procedure to migrate to quorum queues is manual.
+          Please follow the process described here: https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability.
+          Note that this process may take several hours on larger systems, and may cause a degradation in performance at large scale.
+          If you do not wish to enable this feature, set om_enable_rabbitmq_transient_quorum_queue to False.
+      loop: "{{ (rabbitmq_queues.stdout | from_json) if rabbitmq_queues is not skipped else [] }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when:
+        - item.name is search('reply_') or item.name is search('_fanout')
+        - om_enable_rabbitmq_transient_quorum_queue | bool
+
   run_once: true
   when:
     - container_facts.containers['rabbitmq'] is defined
diff --git a/ansible/roles/tacker/templates/tacker.conf.j2 b/ansible/roles/tacker/templates/tacker.conf.j2
index 8f961ac69..ca3bb576e 100644
--- a/ansible/roles/tacker/templates/tacker.conf.j2
+++ b/ansible/roles/tacker/templates/tacker.conf.j2
@@ -82,6 +82,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if tacker_policy_file is defined %}
diff --git a/ansible/roles/trove/templates/trove-guestagent.conf.j2 b/ansible/roles/trove/templates/trove-guestagent.conf.j2
index b9f9e6f77..7b4be3f81 100644
--- a/ansible/roles/trove/templates/trove-guestagent.conf.j2
+++ b/ansible/roles/trove/templates/trove-guestagent.conf.j2
@@ -26,6 +26,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/trove/templates/trove.conf.j2 b/ansible/roles/trove/templates/trove.conf.j2
index bbc7c959a..edc0ab439 100644
--- a/ansible/roles/trove/templates/trove.conf.j2
+++ b/ansible/roles/trove/templates/trove.conf.j2
@@ -87,6 +87,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if enable_osprofiler | bool %}
diff --git a/ansible/roles/watcher/templates/watcher.conf.j2 b/ansible/roles/watcher/templates/watcher.conf.j2
index 844a1c2b1..63cac8352 100644
--- a/ansible/roles/watcher/templates/watcher.conf.j2
+++ b/ansible/roles/watcher/templates/watcher.conf.j2
@@ -73,6 +73,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if watcher_policy_file is defined %}
diff --git a/ansible/roles/zun/templates/zun.conf.j2 b/ansible/roles/zun/templates/zun.conf.j2
index 08221cadc..b46765f4c 100644
--- a/ansible/roles/zun/templates/zun.conf.j2
+++ b/ansible/roles/zun/templates/zun.conf.j2
@@ -143,4 +143,7 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
diff --git a/releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml b/releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml
new file mode 100644
index 000000000..40be10729
--- /dev/null
+++ b/releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml
@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Make use of quorum queues for transient/fanout queues.
+    This helps OpenStack services to be more resilient to message broker
+    failures.
+    This feature is enabled by default.
+    You can disable it by setting the following in your ``globals.yml``:
+    ``om_enable_rabbitmq_transient_quorum_queue: false``.
+    Notice that this disables quorum queues for all transient/fanout message
+    exchanges and likely reduces the resilience of the system.
+    See also `LP#2078339 <https://bugs.launchpad.net/kolla-ansible/+bug/2078339>`_.
diff --git a/tests/upgrade.sh b/tests/upgrade.sh
index 3cd218eb0..e75699ca9 100755
--- a/tests/upgrade.sh
+++ b/tests/upgrade.sh
@@ -13,7 +13,38 @@ function upgrade {
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
     kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
+    # Skip rabbitmq-ha-precheck before the queues are migrated.
+    kolla-ansible prechecks -i ${RAW_INVENTORY} --skip-tags rabbitmq-ha-precheck -vvv &> /tmp/logs/ansible/upgrade-prechecks-pre-rabbitmq
+
+    # NOTE(SvenKieske): As om_enable_rabbitmq_transient_quorum_queue now also
+    # enables quorum_queues for fanout/reply queues in Epoxy, we need
+    # to perform a migration to durable queues.
+    # TODO(SvenKieske): Remove these steps in F Cycle.
+    SERVICE_TAGS="heat,keystone,neutron,nova"
+    if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then
+        SERVICE_TAGS+=",cinder"
+    fi
+    if [[ $SCENARIO == "scenario_nfv" ]]; then
+        SERVICE_TAGS+=",barbican"
+    fi
+    if [[ $SCENARIO == "ironic" ]]; then
+        SERVICE_TAGS+=",ironic"
+    fi
+    if [[ $SCENARIO == "masakari" ]]; then
+        SERVICE_TAGS+=",masakari"
+    fi
+    if [[ $SCENARIO == "ovn" ]] || [[ $SCENARIO == "octavia" ]]; then
+        SERVICE_TAGS+=",octavia"
+    fi
+    if [[ $SCENARIO == "magnum" ]]; then
+        SERVICE_TAGS+=",magnum,designate"
+    fi
+    kolla-ansible stop -i ${RAW_INVENTORY} -vvv --tags $SERVICE_TAGS --yes-i-really-really-mean-it --ignore-missing &> /tmp/logs/ansible/stop
+    kolla-ansible genconfig -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/genconfig
+    kolla-ansible rabbitmq-reset-state -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/rabbitmq-reset-state
+    # Include rabbitmq-ha-precheck this time to confirm all queues have migrated.
     kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-prechecks
+
     kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull-upgrade
     kolla-ansible upgrade -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade