Add oslo.messaging Queue Manager

Adds the variable ``om_enable_queue_manager`` to configure enabling
oslo.messaging Queue Manager for all services which use RabbitMQ. This
is enabled by default.

This is required before we can move away from transient queues
(``rabbit_transient_quorum_queue``) as Queue Manager is needed to avoid
consuming all erlang atoms after some time. It is also a useful feature
for debugging, as queues are now named with hostnames and services
included.

Also setting ``lock_path`` and mounting ``dev/shm`` is dependent on
``om_enable_queue_manager``, these are now enabled too. This will allow
us to backport these features to Caracal and Dalamation without
enforcing the changes right away.

Why is -t added for exec commands?
When using Podman, behavior differs from Docker regarding exec
commands. If -t (which allocates a pseudo-terminal) is not used, the
process runs as a child of podman exec without a PTY. This causes
Python inside the container to assign PGID=0 because the pseudo-terminal
is missing, leading to issues with non-existent paths like /proc/0/....
Adding -t ensures proper PGID assignment, preventing these issues and
ensuring consistent behavior across Docker and Podman.

Side-notes:
The quorum queue precheck needed to be updated as queue manager puts
"_fanout" on the end of the name instead.
We need to override the ``[oslo_messaging_rabbit] processname`` for
services running udner wsgi, as they will otherwise all use the same
processname ``mod_wsgi``. This will cause Permission Errors trying to
access the same file in shared memory as services run with different
users.

Change-Id: Iae5f268e778fbbd2b744dc71a84253ec9e758a99

30 files changed, 123 insertions, 3 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 4e1091414..1e5e2ae9b 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -277,7 +277,7 @@ om_rabbitmq_cacert: "{{ rabbitmq_cacert }}"
 om_enable_rabbitmq_high_availability: false
 # Only enable quorum queues if you disable om_enable_rabbitmq_high_availability
 om_enable_rabbitmq_quorum_queues: true
-om_enable_queue_manager: false
+om_enable_queue_manager: true
 
 ####################
 # Networking options
diff --git a/ansible/roles/aodh/templates/aodh.conf.j2 b/ansible/roles/aodh/templates/aodh.conf.j2
index cbd03db03..9c7ca11d9 100644
--- a/ansible/roles/aodh/templates/aodh.conf.j2
+++ b/ansible/roles/aodh/templates/aodh.conf.j2
@@ -61,6 +61,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'aodh-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'aodh-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/barbican/templates/barbican.conf.j2 b/ansible/roles/barbican/templates/barbican.conf.j2
index 90ce5e903..00de9baa9 100644
--- a/ansible/roles/barbican/templates/barbican.conf.j2
+++ b/ansible/roles/barbican/templates/barbican.conf.j2
@@ -84,6 +84,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/blazar/templates/blazar.conf.j2 b/ansible/roles/blazar/templates/blazar.conf.j2
index df438a62f..e75c91868 100644
--- a/ansible/roles/blazar/templates/blazar.conf.j2
+++ b/ansible/roles/blazar/templates/blazar.conf.j2
@@ -55,6 +55,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/ceilometer/templates/ceilometer.conf.j2 b/ansible/roles/ceilometer/templates/ceilometer.conf.j2
index f03100e3e..c39e73ddb 100644
--- a/ansible/roles/ceilometer/templates/ceilometer.conf.j2
+++ b/ansible/roles/ceilometer/templates/ceilometer.conf.j2
@@ -36,6 +36,9 @@ ca_file = /etc/ceilometer/vmware_ca
 transport_url = {{ notify_transport_url }}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2
index 732db76a5..372527b23 100644
--- a/ansible/roles/cinder/templates/cinder.conf.j2
+++ b/ansible/roles/cinder/templates/cinder.conf.j2
@@ -79,6 +79,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'cinder-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'cinder-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
index 413eb5c9c..0526e1133 100644
--- a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
+++ b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
@@ -45,6 +45,12 @@ policy_file = {{ cloudkitty_policy_file }}
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'cloudkitty-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'cloudkitty-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/cyborg/templates/cyborg.conf.j2 b/ansible/roles/cyborg/templates/cyborg.conf.j2
index ba00d270d..58b1169bf 100644
--- a/ansible/roles/cyborg/templates/cyborg.conf.j2
+++ b/ansible/roles/cyborg/templates/cyborg.conf.j2
@@ -70,6 +70,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/designate/templates/designate.conf.j2 b/ansible/roles/designate/templates/designate.conf.j2
index 27cc901e4..2b61fce59 100644
--- a/ansible/roles/designate/templates/designate.conf.j2
+++ b/ansible/roles/designate/templates/designate.conf.j2
@@ -93,6 +93,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/glance/templates/glance-api.conf.j2 b/ansible/roles/glance/templates/glance-api.conf.j2
index b4766cb9b..6bdc35e36 100644
--- a/ansible/roles/glance/templates/glance-api.conf.j2
+++ b/ansible/roles/glance/templates/glance-api.conf.j2
@@ -125,6 +125,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/heat/templates/heat.conf.j2 b/ansible/roles/heat/templates/heat.conf.j2
index 793c42d10..2121996b2 100644
--- a/ansible/roles/heat/templates/heat.conf.j2
+++ b/ansible/roles/heat/templates/heat.conf.j2
@@ -72,6 +72,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'heat-api' or service_name == 'heat-api-cfn' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'heat-api' or service_name == 'heat-api-cfn' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 b/ansible/roles/ironic/templates/ironic-inspector.conf.j2
index 16744250f..d45f40d5f 100644
--- a/ansible/roles/ironic/templates/ironic-inspector.conf.j2
+++ b/ansible/roles/ironic/templates/ironic-inspector.conf.j2
@@ -13,6 +13,9 @@ transport_url = {{ rpc_transport_url }}
 transport_url = {{ notify_transport_url }}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2
index 94096f6ff..6adee1bca 100644
--- a/ansible/roles/ironic/templates/ironic.conf.j2
+++ b/ansible/roles/ironic/templates/ironic.conf.j2
@@ -43,6 +43,10 @@ backend = collector
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+processname = {{ service_name }}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'ironic-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/keystone/templates/keystone.conf.j2 b/ansible/roles/keystone/templates/keystone.conf.j2
index af43cfc93..cf44e0991 100644
--- a/ansible/roles/keystone/templates/keystone.conf.j2
+++ b/ansible/roles/keystone/templates/keystone.conf.j2
@@ -60,6 +60,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'keystone-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'keystone' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/magnum/templates/magnum.conf.j2 b/ansible/roles/magnum/templates/magnum.conf.j2
index c9c720af8..ce97de7ab 100644
--- a/ansible/roles/magnum/templates/magnum.conf.j2
+++ b/ansible/roles/magnum/templates/magnum.conf.j2
@@ -127,6 +127,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/manila/templates/manila.conf.j2 b/ansible/roles/manila/templates/manila.conf.j2
index da78f6a12..2383380e3 100644
--- a/ansible/roles/manila/templates/manila.conf.j2
+++ b/ansible/roles/manila/templates/manila.conf.j2
@@ -57,6 +57,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/masakari/templates/masakari.conf.j2 b/ansible/roles/masakari/templates/masakari.conf.j2
index b96e6c6d1..dff6040b4 100644
--- a/ansible/roles/masakari/templates/masakari.conf.j2
+++ b/ansible/roles/masakari/templates/masakari.conf.j2
@@ -51,6 +51,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'masakari-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'masakari-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/mistral/templates/mistral.conf.j2 b/ansible/roles/mistral/templates/mistral.conf.j2
index 5ce7a4a0e..c7f03475f 100644
--- a/ansible/roles/mistral/templates/mistral.conf.j2
+++ b/ansible/roles/mistral/templates/mistral.conf.j2
@@ -73,6 +73,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/neutron/templates/neutron.conf.j2 b/ansible/roles/neutron/templates/neutron.conf.j2
index 7b5aea818..3b7e4063b 100644
--- a/ansible/roles/neutron/templates/neutron.conf.j2
+++ b/ansible/roles/neutron/templates/neutron.conf.j2
@@ -132,6 +132,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/nova-cell/tasks/discover_computes.yml b/ansible/roles/nova-cell/tasks/discover_computes.yml
index 559be6aaa..9b6a1696f 100644
--- a/ansible/roles/nova-cell/tasks/discover_computes.yml
+++ b/ansible/roles/nova-cell/tasks/discover_computes.yml
@@ -14,7 +14,7 @@
     - name: Discover nova hosts
       become: true
       command: >
-        {{ kolla_container_engine }} exec nova_conductor nova-manage cell_v2 discover_hosts --by-service --cell_uuid {{ nova_cell_settings.cell_uuid }}
+        {{ kolla_container_engine }} exec -t nova_conductor nova-manage cell_v2 discover_hosts --by-service --cell_uuid {{ nova_cell_settings.cell_uuid }}
       changed_when: False
 
   # Delegate to a cell conductor.
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index cdacedf4a..a52132e31 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -189,6 +189,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == "nova-compute-ironic" %}
+hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/nova/templates/nova.conf.j2 b/ansible/roles/nova/templates/nova.conf.j2
index 3a3551956..0b7680f81 100644
--- a/ansible/roles/nova/templates/nova.conf.j2
+++ b/ansible/roles/nova/templates/nova.conf.j2
@@ -139,6 +139,12 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'nova-api' or service_name == 'nova-metadata' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'nova-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/octavia/templates/octavia.conf.j2 b/ansible/roles/octavia/templates/octavia.conf.j2
index 074d1d1f5..f5d06e951 100644
--- a/ansible/roles/octavia/templates/octavia.conf.j2
+++ b/ansible/roles/octavia/templates/octavia.conf.j2
@@ -123,6 +123,12 @@ rpc_thread_pool_size = 2
 transport_url = {{ notify_transport_url }}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'octavia-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'octavia-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/rabbitmq/tasks/precheck.yml b/ansible/roles/rabbitmq/tasks/precheck.yml
index ea9f7ee71..3df9548b8 100644
--- a/ansible/roles/rabbitmq/tasks/precheck.yml
+++ b/ansible/roles/rabbitmq/tasks/precheck.yml
@@ -150,7 +150,7 @@
       loop_control:
         label: "{{ item.name }}"
       # TODO(mattcress): remove skipping reply and fanout once https://review.opendev.org/c/openstack/oslo.messaging/+/888479 is merged.
-      when: not (item.name is search('reply_') or item.name is search('_fanout_'))
+      when: not (item.name is search('reply_') or item.name is search('_fanout'))
 
   run_once: true
   when:
diff --git a/ansible/roles/tacker/templates/tacker.conf.j2 b/ansible/roles/tacker/templates/tacker.conf.j2
index ed091f094..8f961ac69 100644
--- a/ansible/roles/tacker/templates/tacker.conf.j2
+++ b/ansible/roles/tacker/templates/tacker.conf.j2
@@ -69,6 +69,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/trove/templates/trove-guestagent.conf.j2 b/ansible/roles/trove/templates/trove-guestagent.conf.j2
index ca29734aa..b9f9e6f77 100644
--- a/ansible/roles/trove/templates/trove-guestagent.conf.j2
+++ b/ansible/roles/trove/templates/trove-guestagent.conf.j2
@@ -17,6 +17,9 @@ user_domain_name = {{ default_user_domain_name }}
 username = {{ trove_keystone_user }}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
diff --git a/ansible/roles/trove/templates/trove.conf.j2 b/ansible/roles/trove/templates/trove.conf.j2
index 6dd3b72f0..cc85f0c67 100644
--- a/ansible/roles/trove/templates/trove.conf.j2
+++ b/ansible/roles/trove/templates/trove.conf.j2
@@ -75,6 +75,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/watcher/templates/watcher.conf.j2 b/ansible/roles/watcher/templates/watcher.conf.j2
index 2afa97cb3..844a1c2b1 100644
--- a/ansible/roles/watcher/templates/watcher.conf.j2
+++ b/ansible/roles/watcher/templates/watcher.conf.j2
@@ -60,6 +60,9 @@ driver = noop
 {% endif %}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/ansible/roles/zun/templates/zun.conf.j2 b/ansible/roles/zun/templates/zun.conf.j2
index 30e456285..08221cadc 100644
--- a/ansible/roles/zun/templates/zun.conf.j2
+++ b/ansible/roles/zun/templates/zun.conf.j2
@@ -127,6 +127,12 @@ docker_remote_api_port = 2375
 cni_daemon_port = {{ zun_cni_daemon_port }}
 
 [oslo_messaging_rabbit]
+{% if om_enable_queue_manager | bool %}
+use_queue_manager = true
+{% if service_name == 'zun-api' %}
+processname = {{ service_name }}
+{% endif %}
+{% endif %}
 heartbeat_in_pthread = {{ service_name == 'zun-api' }}
 {% if om_enable_rabbitmq_tls | bool %}
 ssl = true
diff --git a/releasenotes/notes/add-queue-manager-3ce79655ac37c345.yaml b/releasenotes/notes/add-queue-manager-3ce79655ac37c345.yaml
new file mode 100644
index 000000000..0a2e1edbb
--- /dev/null
+++ b/releasenotes/notes/add-queue-manager-3ce79655ac37c345.yaml
@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Added support for using oslo.messaging Queue Manager to all services which
+    use RabbitMQ. This is enabled by default, but can be configured using
+    ``om_enable_queue_manager``.
+upgrade:
+  - |
+    Added support for using oslo.messaging Queue Manager to all services which
+    use RabbitMQ. This is enabled by default, but can be configured using
+    ``om_enable_queue_manager``.