Merge "Support multiple Nova Compute Ironic instances"

16 files changed, 274 insertions, 12 deletions

diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml
index 6979408e9..790190fb0 100644
--- a/ansible/roles/nova-cell/defaults/main.yml
+++ b/ansible/roles/nova-cell/defaults/main.yml
@@ -72,6 +72,8 @@ nova_cell_services:
     volumes: "{{ nova_compute_ironic_default_volumes + nova_compute_ironic_extra_volumes }}"
     dimensions: "{{ nova_compute_ironic_dimensions }}"
     healthcheck: "{{ nova_compute_ironic_healthcheck }}"
+    iterate: "{{ nova_multi_compute_ironic_config | length > 0 }}"
+    iterate_var: "{{ nova_multi_compute_ironic_config | length }}"
 
 ####################
 # Config Validate
@@ -501,7 +503,7 @@ nova_compute_default_volumes:
   - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
   - "{{ kolla_dev_repos_directory ~ '/nova:/dev-mode/nova' if nova_dev_mode | bool else '' }}"
 nova_compute_ironic_default_volumes:
-  - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
+  - "{{ node_config_directory }}/nova-compute-ironic{{ '-' ~ item if item|default(0)|int > 0 else '' }}/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "kolla_logs:/var/log/kolla/"
   - "{{ '/dev/shm:/dev/shm' }}"
@@ -675,6 +677,17 @@ nova_libvirt_cleanup_remove_volumes: false
 # Database
 nova_cell_database_enable_tls_internal: "{{ database_enable_tls_internal | bool }}"
 
+##############################
+# Nova Compute Ironic scaling
+##############################
+
+# The following option can be used to deploy multiple instances of Nova
+# Compute Ironic per host. This is useful for deployments with a large
+# number of baremetal nodes (>200). Please see the advanced documentation
+# section for further details.
+
+nova_multi_compute_ironic_config: []
+
 ###################
 # Copy certificates
 ###################
diff --git a/ansible/roles/nova-cell/handlers/main.yml b/ansible/roles/nova-cell/handlers/main.yml
index a67a1231f..c6ac28da9 100644
--- a/ansible/roles/nova-cell/handlers/main.yml
+++ b/ansible/roles/nova-cell/handlers/main.yml
@@ -162,16 +162,21 @@
   vars:
     service_name: "nova-compute-ironic"
     service: "{{ nova_cell_services[service_name] }}"
+    item: "{{ changed_container.item }}"
+    container_name_postfix: "{{ '' if changed_container == 'legacy' else '_' ~ item }}"
   become: true
   kolla_container:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
+    name: "{{ service.container_name + container_name_postfix }}"
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes | reject('equalto', '') | list }}"
     dimensions: "{{ service.dimensions }}"
     healthcheck: "{{ service.healthcheck | default(omit) }}"
+  loop_control:
+    loop_var: changed_container
+  loop: "{{ nova_compute_ironic_changed_containers | default(['legacy']) }}"
 
 # nova-compute-fake is special. It will start multi numbers of container
 # so put all variables here rather than defaults/main.yml file
diff --git a/ansible/roles/nova-cell/tasks/cleanup.yml b/ansible/roles/nova-cell/tasks/cleanup.yml
new file mode 100644
index 000000000..0f0952823
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/cleanup.yml
@@ -0,0 +1,26 @@
+---
+- name: Stop and remove containers for unmanaged services
+  become: true
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  kolla_container:
+    action: "stop_and_remove_container"
+    name: "{{ service.container_name ~ '_' ~ item }}"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
+
+- name: Removing config for unmanaged services
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/{{ service_name ~ '-' ~ item }}"
+    state: "absent"
+  become: true
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
diff --git a/ansible/roles/nova-cell/tasks/config-compute-ironic.yml b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml
new file mode 100644
index 000000000..0f1c05a9f
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml
@@ -0,0 +1,67 @@
+---
+- name: "Ensuring config directories exist for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+
+- name: "Copying over config.json files for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.template:
+    src: "nova-compute-ironic.json.j2"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/config.json"
+    mode: "0660"
+  become: true
+
+- name: "Get Nova Compute Ironic multi-instance configuration for instance {{ item }}"
+  ansible.builtin.set_fact:
+    nova_multi_compute_ironic_instance_config: "{{ nova_multi_compute_ironic_config[(item | int) - 1] }}"
+
+- name: "Generate config files for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+    nova_compute_shard_key: "{{ nova_multi_compute_ironic_instance_config['shard_key'] | default('default') }}"
+    nova_compute_conductor_group: "{{ nova_multi_compute_ironic_instance_config['conductor_group'] | default('default') }}"
+    nova_compute_ironic_custom_host: "{{ nova_multi_compute_ironic_instance_config['custom_host'] | default('') }}"
+    nova_compute_ironic_id: "{{ item }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/nova.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}-{{ item }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}-{{ item }}.conf"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/nova.conf"
+    mode: "0660"
+  become: true
+
+- name: "Copying over existing policy file for item {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.template:
+    src: "{{ nova_policy_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/{{ nova_policy_file }}"
+    mode: "0660"
+  become: true
+  when:
+    - nova_policy_file is defined
+
+- name: "Copying over vendordata file for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.copy:
+    src: "{{ vendordata_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/vendordata.json"
+    mode: "0660"
+  become: true
+  when:
+    - vendordata_file_path is defined
diff --git a/ansible/roles/nova-cell/tasks/config.yml b/ansible/roles/nova-cell/tasks/config.yml
index 87104f89e..7f0cfca22 100644
--- a/ansible/roles/nova-cell/tasks/config.yml
+++ b/ansible/roles/nova-cell/tasks/config.yml
@@ -7,6 +7,8 @@
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0770"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over TLS certificates
@@ -58,6 +60,8 @@
     src: "{{ item.key }}.json.j2"
     dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
     mode: "0660"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over nova.conf
@@ -76,8 +80,18 @@
     mode: "0660"
   when:
     - item.key in nova_cell_services_require_nova_conf
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
+- name: Copying over config for Nova compute Ironic multi-instance
+  vars:
+    service: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.include_tasks: config-compute-ironic.yml
+  loop: "{{ range(1, nova_multi_compute_ironic_config | length + 1) | list }}"
+  when:
+    - nova_multi_compute_ironic_config | length > 0
+    - inventory_hostname in groups[service.group ~ '-' ~ item]
+
 - name: Copying over Nova compute provider config
   become: true
   vars:
@@ -196,6 +210,7 @@
   when:
     - nova_policy_file is defined
     - item.key in nova_cell_services_require_policy_json
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over vendordata file to containers
@@ -209,6 +224,7 @@
   when:
     - vendordata_file_path is defined
     - service | service_enabled_and_mapped_to_host
+    - item != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_items:
     - nova-compute
     - nova-compute-ironic
diff --git a/ansible/roles/nova-cell/tasks/deploy.yml b/ansible/roles/nova-cell/tasks/deploy.yml
index 2a416d8d4..288531557 100644
--- a/ansible/roles/nova-cell/tasks/deploy.yml
+++ b/ansible/roles/nova-cell/tasks/deploy.yml
@@ -6,6 +6,10 @@
 - name: Import tasks from version-check.yml
   ansible.builtin.import_tasks: version-check.yml
 
+# NOTE(dougszu): Currently only covers nova-compute-ironic
+- name: Cleanup stale configuration
+  ansible.builtin.import_tasks: cleanup.yml
+
 - name: Import tasks from config-host.yml
   ansible.builtin.import_tasks: config-host.yml
 
diff --git a/ansible/roles/nova-cell/tasks/precheck.yml b/ansible/roles/nova-cell/tasks/precheck.yml
index a509e790c..9364c619e 100644
--- a/ansible/roles/nova-cell/tasks/precheck.yml
+++ b/ansible/roles/nova-cell/tasks/precheck.yml
@@ -30,6 +30,20 @@
     - groups[nova_cell_compute_group] | length < 1
     - not nova_compute_ironic.enabled | bool
 
+- name: Checking nova multi-compute-ironic inventory
+  vars:
+    nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.assert:
+    msg: >
+      Each nova-compute-ironic-N group must contain a single host.
+    that:
+      - groups[nova_cell_compute_ironic_group ~ '-' ~ item] | length == 1
+  loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}"
+  run_once: true
+  when:
+    - nova_compute_ironic.iterate | bool
+    - nova_compute_ironic.enabled | bool
+
 - name: Checking free port for Nova NoVNC Proxy
   vars:
     service: "{{ nova_cell_services['nova-novncproxy'] }}"
diff --git a/ansible/roles/nova-cell/tasks/reload.yml b/ansible/roles/nova-cell/tasks/reload.yml
index 5724effdf..5f5c05b4e 100644
--- a/ansible/roles/nova-cell/tasks/reload.yml
+++ b/ansible/roles/nova-cell/tasks/reload.yml
@@ -37,4 +37,28 @@
   when:
     - kolla_action == 'upgrade'
     - service | service_enabled_and_mapped_to_host
+    - item != 'nova-compute-ironic' or not service.iterate | bool
   with_items: "{{ nova_cell_services_require_nova_conf }}"
+
+- name: Reload nova-compute-ironic to remove RPC version cap (iterated)
+  vars:
+    nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  become: true
+  kolla_container:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ nova_compute_ironic.container_name ~ '_' ~ item }}"
+    image: "{{ nova_compute_ironic.image }}"
+    environment: "{{ nova_compute_ironic.environment | default(omit) }}"
+    pid_mode: "{{ nova_compute_ironic.pid_mode | default('') }}"
+    ipc_mode: "{{ nova_compute_ironic.ipc_mode | default(omit) }}"
+    privileged: "{{ nova_compute_ironic.privileged | default(False) }}"
+    volumes: "{{ nova_compute_ironic.volumes | reject('equalto', '') | list }}"
+    dimensions: "{{ nova_compute_ironic.dimensions }}"
+    healthcheck: "{{ nova_compute_ironic.healthcheck | default(omit) }}"
+  when:
+    - kolla_action == 'upgrade'
+    - nova_compute_ironic | service_enabled
+    - nova_compute_ironic.iterate | bool
+    - inventory_hostname in groups[nova_compute_ironic.group ~ '-' ~ item]
+  loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}"
diff --git a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
index d930f8dea..eb7e80e83 100644
--- a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
+++ b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
@@ -13,13 +13,14 @@
       {{ virt_computes_in_batch |
          map('extract', hostvars, ['ansible_facts', 'nodename']) |
          list }}
-    # For ironic, use {{ansible_facts.hostname}}-ironic since this is what we
-    # configure for [DEFAULT] host in nova.conf.
     ironic_compute_service_hosts: >-
       {{ ironic_computes_in_batch |
-         map('extract', hostvars) | json_query('[].nova_compute_ironic_custom_host || [].ansible_facts.hostname') |
-         map('regex_replace', '^(.*)$', '\1-ironic') |
-         list }}
+         map('extract', hostvars) |
+         json_query(
+           '{classic: [].nova_compute_ironic_custom_host || [].ansible_facts.hostname,
+             multi: [].group_names}') |
+         get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }}
+
     expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}"
   block:
     - name: Waiting for nova-compute services to register themselves
@@ -73,13 +74,19 @@
         failed_compute_service_hosts: >-
           {{ expected_compute_service_hosts | difference(nova_compute_service_hosts) | list }}
         # Whether any compute services failed on this host.
+        ironic_compute_service_config_for_host:
+          classic: "{{ [nova_compute_ironic_custom_host | default(ansible_facts.hostname)] if inventory_hostname in ironic_computes_in_batch else [] }}"
+          multi: "{{ [group_names] }}"
+        expected_ironic_compute_services: >-
+          {{ ironic_compute_service_config_for_host | get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }}
         any_failed_services: >-
           {{ (ansible_facts.nodename | default()) in failed_compute_service_hosts or
-             ((ansible_facts.hostname | default()) ~ "-ironic") in failed_compute_service_hosts }}
+             expected_ironic_compute_services | intersect(failed_compute_service_hosts) | length > 0 }}
+
       ansible.builtin.fail:
         msg: >-
-          The Nova compute service failed to register itself on the following
-          hosts: {{ failed_compute_service_hosts | join(',') }}
+          The following Nova compute services failed to register
+          themselves: {{ failed_compute_service_hosts | join(',') }}
       when: >-
         any_failed_services or
         (nova_compute_registration_fatal | bool and
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index 2950dfcc4..7d30dcfd0 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -8,9 +8,27 @@ state_path = /var/lib/nova
 
 allow_resize_to_same_host = true
 
+{% set nova_compute_shard_key = nova_compute_shard_key | default("") %}
+{% set nova_compute_conductor_group = nova_compute_conductor_group | default("") %}
+{% set nova_compute_ironic_custom_host = nova_compute_ironic_custom_host | default("") %}
 {% if service_name == "nova-compute-ironic" %}
-host={{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+{%     if nova_compute_ironic_custom_host | length > 0 %}
+{#         NOTE(dougszu): This is provided for backwards compatibility #}
+{%         set host_value = nova_compute_ironic_custom_host ~ "-ironic" %}
+{%     elif nova_compute_conductor_group | length > 0 or nova_compute_shard_key | length > 0 %}
+{%         set host_value = (nova_compute_conductor_group ~ "-" ~ nova_compute_shard_key) ~ "-ironic" %}
+{%     else %}
+{#         NOTE(dougszu): This is to support legacy behaviour #}
+{%         set host_value = ansible_facts.hostname ~ "-ironic" %}
+{%     endif %}
+host = {{ host_value }}
+
+{%     if nova_compute_ironic_id is defined %}
+log_file = /var/log/kolla/nova/nova-compute-ironic-{{ nova_compute_ironic_id }}.log
+{%     else %}
 log_file = /var/log/kolla/nova/nova-compute-ironic.log
+{%     endif %}
+
 compute_driver = ironic.IronicDriver
 ram_allocation_ratio = 1.0
 reserved_host_memory_mb = 0
@@ -101,6 +119,19 @@ project_name = service
 user_domain_name = {{ default_user_domain_name }}
 project_domain_name = {{ default_project_domain_name }}
 endpoint_override = {{ ironic_internal_endpoint }}/v1
+
+{%     if nova_compute_shard_key is not in ['default', ''] %}
+shard = {{ nova_compute_shard_key }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in ['default', ''] %}
+conductor_group = {{ nova_compute_conductor_group }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in ['default', ''] and nova_compute_shard_key is in ['default', ''] %}
+{# NOTE(dougszu): In this case only, Nova Compute Ironic won't start unless the peer_list #}
+{# is populated. peer_list is a deprecated config option and should never have more than #}
+{# one host in it. This appears to be a bug. #}
+peer_list = {{ host_value }}
+{%     endif %}
 {% endif %}
 
 [oslo_concurrency]
@@ -180,7 +211,7 @@ driver = noop
 [oslo_messaging_rabbit]
 use_queue_manager = true
 {% if service_name == "nova-compute-ironic" %}
-hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+hostname = {{ host_value }}
 {% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
diff --git a/ansible/roles/service-cert-copy/tasks/iterated.yml b/ansible/roles/service-cert-copy/tasks/iterated.yml
new file mode 100644
index 000000000..4da5741fe
--- /dev/null
+++ b/ansible/roles/service-cert-copy/tasks/iterated.yml
@@ -0,0 +1,10 @@
+---
+- name: "Copying over extra CA certificates for {{ project_name }}"
+  become: true
+  ansible.builtin.copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ outer_item.key }}-{{ item }}/ca-certificates"
+    mode: "0644"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - inventory_hostname in groups[service.group ~ '-' ~ item]
diff --git a/ansible/roles/service-cert-copy/tasks/main.yml b/ansible/roles/service-cert-copy/tasks/main.yml
index d96bf0ca6..ef7ac6717 100644
--- a/ansible/roles/service-cert-copy/tasks/main.yml
+++ b/ansible/roles/service-cert-copy/tasks/main.yml
@@ -7,8 +7,21 @@
     mode: "0644"
   when:
     - kolla_copy_ca_into_containers | bool
+    - not (item.value.iterate | default(False)) | bool
   with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
 
+# NOTE(dougszu): For current iterable services, we only need CA certs
+- name: "Check containers that require iteration for {{ kolla_role_name | default(project_name) }}"
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ project_services | select_services_enabled_and_mapped_to_host | dict2items }}"
+  loop_control:
+    loop_var: outer_item
+  when:
+    - (service.iterate | default(False)) | bool
+    - kolla_copy_ca_into_containers | bool
+
 - name: "Copying over backend internal TLS certificate for {{ project_name }}"
   vars:
     certs:
diff --git a/ansible/roles/service-check-containers/tasks/iterated.yml b/ansible/roles/service-check-containers/tasks/iterated.yml
index bd266b115..c4f28e6d4 100644
--- a/ansible/roles/service-check-containers/tasks/iterated.yml
+++ b/ansible/roles/service-check-containers/tasks/iterated.yml
@@ -23,6 +23,12 @@
     command: "{{ service.command | default(omit) }}"
     cgroupns_mode: "{{ service.cgroupns_mode | default(omit) }}"
   loop: "{{ range(1, (iterate_count | int) + 1) | list }}"
+  when:
+    # NOTE(dougszu): Due to issues with nova-compute-ironic HA, we can't have
+    # more than one instance for a given 'item'. Eg. nova_compute_ironic_1
+    # should never run on more than one host.
+    # See: https://specs.openstack.org/openstack/nova-specs/specs/2024.1/implemented/ironic-shards.html
+    - service_name != 'nova-compute-ironic' or inventory_hostname in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
   register: container_check
 
 - name: "Store changed containers to pass to the handler"
diff --git a/ansible/roles/service-precheck/tasks/iterated.yml b/ansible/roles/service-precheck/tasks/iterated.yml
new file mode 100644
index 000000000..d9157ddce
--- /dev/null
+++ b/ansible/roles/service-precheck/tasks/iterated.yml
@@ -0,0 +1,13 @@
+---
+- name: "Validate inventory groups (iterated) for {{ project_name }}"
+  vars:
+    service_name: "{{ outer_item.key }}"
+    service: "{{ outer_item.value }}"
+    service_group: "{{ service.group ~ '-' ~ item }}"
+  ansible.builtin.fail:
+    msg: >-
+      Ansible inventory does not contain the expected group {{ service_group }}
+      for service {{ service_name }} in {{ project_name }}.
+  when:
+    - service_group not in groups
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
diff --git a/ansible/roles/service-precheck/tasks/main.yml b/ansible/roles/service-precheck/tasks/main.yml
index 58448555e..81b90f6a5 100644
--- a/ansible/roles/service-precheck/tasks/main.yml
+++ b/ansible/roles/service-precheck/tasks/main.yml
@@ -9,7 +9,19 @@
       for service {{ service_name }} in {{ project_name }}.
   loop: "{{ query('dict', service_precheck_services) }}"
   when:
+    - not (service.iterate | default(False)) | bool
     - "'group' in service"
     - service.group not in groups
   loop_control:
     label: "{{ service_name }}"
+
+- name: Include tasks
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ query('dict', service_precheck_services) }}"
+  loop_control:
+    loop_var: outer_item
+  when:
+    - (service.iterate | default(False)) | bool
+    - "'group' in service"
diff --git a/ansible/roles/service-stop/tasks/iterated.yml b/ansible/roles/service-stop/tasks/iterated.yml
index f25bf0c26..cf4e8e489 100644
--- a/ansible/roles/service-stop/tasks/iterated.yml
+++ b/ansible/roles/service-stop/tasks/iterated.yml
@@ -11,4 +11,5 @@
     ignore_missing: "{{ kolla_action_stop_ignore_missing | bool }}"
   when:
     - service.container_name not in skip_stop_containers
+    - service_name != 'nova-compute-ironic' or inventory_hostname in groups[service.group ~ '-' ~ item]
   loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"