Support multiple Nova Compute Ironic instances

Adds support for deploying multiple instances of the Nova
Compute Ironic service on the same host. This is useful
in large baremetal deployments, where the sharding and/or
conductor group feature is used to scale out the service.

A further patch to support deploying multiple Ironic
conductor instances will follow.

Co-Authored-By: Bartosz Bezak <bartosz@stackhpc.com>
Co-Authored-By: Bertrand Lanson <bertrand.lanson@protonmail.com>
Change-Id: Ibbddfd87e831a3775c3ed66c80b18e5070cedc90
Signed-off-by: Doug Szumski <doug@stackhpc.com>

27 files changed, 627 insertions, 19 deletions

diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml
index 6979408e9..790190fb0 100644
--- a/ansible/roles/nova-cell/defaults/main.yml
+++ b/ansible/roles/nova-cell/defaults/main.yml
@@ -72,6 +72,8 @@ nova_cell_services:
     volumes: "{{ nova_compute_ironic_default_volumes + nova_compute_ironic_extra_volumes }}"
     dimensions: "{{ nova_compute_ironic_dimensions }}"
     healthcheck: "{{ nova_compute_ironic_healthcheck }}"
+    iterate: "{{ nova_multi_compute_ironic_config | length > 0 }}"
+    iterate_var: "{{ nova_multi_compute_ironic_config | length }}"
 
 ####################
 # Config Validate
@@ -501,7 +503,7 @@ nova_compute_default_volumes:
   - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
   - "{{ kolla_dev_repos_directory ~ '/nova:/dev-mode/nova' if nova_dev_mode | bool else '' }}"
 nova_compute_ironic_default_volumes:
-  - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
+  - "{{ node_config_directory }}/nova-compute-ironic{{ '-' ~ item if item|default(0)|int > 0 else '' }}/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "kolla_logs:/var/log/kolla/"
   - "{{ '/dev/shm:/dev/shm' }}"
@@ -675,6 +677,17 @@ nova_libvirt_cleanup_remove_volumes: false
 # Database
 nova_cell_database_enable_tls_internal: "{{ database_enable_tls_internal | bool }}"
 
+##############################
+# Nova Compute Ironic scaling
+##############################
+
+# The following option can be used to deploy multiple instances of Nova
+# Compute Ironic per host. This is useful for deployments with a large
+# number of baremetal nodes (>200). Please see the advanced documentation
+# section for further details.
+
+nova_multi_compute_ironic_config: []
+
 ###################
 # Copy certificates
 ###################
diff --git a/ansible/roles/nova-cell/handlers/main.yml b/ansible/roles/nova-cell/handlers/main.yml
index a67a1231f..c6ac28da9 100644
--- a/ansible/roles/nova-cell/handlers/main.yml
+++ b/ansible/roles/nova-cell/handlers/main.yml
@@ -162,16 +162,21 @@
   vars:
     service_name: "nova-compute-ironic"
     service: "{{ nova_cell_services[service_name] }}"
+    item: "{{ changed_container.item }}"
+    container_name_postfix: "{{ '' if changed_container == 'legacy' else '_' ~ item }}"
   become: true
   kolla_container:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
+    name: "{{ service.container_name + container_name_postfix }}"
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes | reject('equalto', '') | list }}"
     dimensions: "{{ service.dimensions }}"
     healthcheck: "{{ service.healthcheck | default(omit) }}"
+  loop_control:
+    loop_var: changed_container
+  loop: "{{ nova_compute_ironic_changed_containers | default(['legacy']) }}"
 
 # nova-compute-fake is special. It will start multi numbers of container
 # so put all variables here rather than defaults/main.yml file
diff --git a/ansible/roles/nova-cell/tasks/cleanup.yml b/ansible/roles/nova-cell/tasks/cleanup.yml
new file mode 100644
index 000000000..0f0952823
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/cleanup.yml
@@ -0,0 +1,26 @@
+---
+- name: Stop and remove containers for unmanaged services
+  become: true
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  kolla_container:
+    action: "stop_and_remove_container"
+    name: "{{ service.container_name ~ '_' ~ item }}"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
+
+- name: Removing config for unmanaged services
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/{{ service_name ~ '-' ~ item }}"
+    state: "absent"
+  become: true
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
diff --git a/ansible/roles/nova-cell/tasks/config-compute-ironic.yml b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml
new file mode 100644
index 000000000..0f1c05a9f
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml
@@ -0,0 +1,67 @@
+---
+- name: "Ensuring config directories exist for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+
+- name: "Copying over config.json files for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.template:
+    src: "nova-compute-ironic.json.j2"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/config.json"
+    mode: "0660"
+  become: true
+
+- name: "Get Nova Compute Ironic multi-instance configuration for instance {{ item }}"
+  ansible.builtin.set_fact:
+    nova_multi_compute_ironic_instance_config: "{{ nova_multi_compute_ironic_config[(item | int) - 1] }}"
+
+- name: "Generate config files for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+    nova_compute_shard_key: "{{ nova_multi_compute_ironic_instance_config['shard_key'] | default('default') }}"
+    nova_compute_conductor_group: "{{ nova_multi_compute_ironic_instance_config['conductor_group'] | default('default') }}"
+    nova_compute_ironic_custom_host: "{{ nova_multi_compute_ironic_instance_config['custom_host'] | default('') }}"
+    nova_compute_ironic_id: "{{ item }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/nova.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}-{{ item }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}-{{ item }}.conf"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/nova.conf"
+    mode: "0660"
+  become: true
+
+- name: "Copying over existing policy file for item {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.template:
+    src: "{{ nova_policy_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/{{ nova_policy_file }}"
+    mode: "0660"
+  become: true
+  when:
+    - nova_policy_file is defined
+
+- name: "Copying over vendordata file for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.copy:
+    src: "{{ vendordata_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/vendordata.json"
+    mode: "0660"
+  become: true
+  when:
+    - vendordata_file_path is defined
diff --git a/ansible/roles/nova-cell/tasks/config.yml b/ansible/roles/nova-cell/tasks/config.yml
index 87104f89e..7f0cfca22 100644
--- a/ansible/roles/nova-cell/tasks/config.yml
+++ b/ansible/roles/nova-cell/tasks/config.yml
@@ -7,6 +7,8 @@
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0770"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over TLS certificates
@@ -58,6 +60,8 @@
     src: "{{ item.key }}.json.j2"
     dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
     mode: "0660"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over nova.conf
@@ -76,8 +80,18 @@
     mode: "0660"
   when:
     - item.key in nova_cell_services_require_nova_conf
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
+- name: Copying over config for Nova compute Ironic multi-instance
+  vars:
+    service: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.include_tasks: config-compute-ironic.yml
+  loop: "{{ range(1, nova_multi_compute_ironic_config | length + 1) | list }}"
+  when:
+    - nova_multi_compute_ironic_config | length > 0
+    - inventory_hostname in groups[service.group ~ '-' ~ item]
+
 - name: Copying over Nova compute provider config
   become: true
   vars:
@@ -196,6 +210,7 @@
   when:
     - nova_policy_file is defined
     - item.key in nova_cell_services_require_policy_json
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over vendordata file to containers
@@ -209,6 +224,7 @@
   when:
     - vendordata_file_path is defined
     - service | service_enabled_and_mapped_to_host
+    - item != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_items:
     - nova-compute
     - nova-compute-ironic
diff --git a/ansible/roles/nova-cell/tasks/deploy.yml b/ansible/roles/nova-cell/tasks/deploy.yml
index 2a416d8d4..288531557 100644
--- a/ansible/roles/nova-cell/tasks/deploy.yml
+++ b/ansible/roles/nova-cell/tasks/deploy.yml
@@ -6,6 +6,10 @@
 - name: Import tasks from version-check.yml
   ansible.builtin.import_tasks: version-check.yml
 
+# NOTE(dougszu): Currently only covers nova-compute-ironic
+- name: Cleanup stale configuration
+  ansible.builtin.import_tasks: cleanup.yml
+
 - name: Import tasks from config-host.yml
   ansible.builtin.import_tasks: config-host.yml
 
diff --git a/ansible/roles/nova-cell/tasks/precheck.yml b/ansible/roles/nova-cell/tasks/precheck.yml
index a509e790c..9364c619e 100644
--- a/ansible/roles/nova-cell/tasks/precheck.yml
+++ b/ansible/roles/nova-cell/tasks/precheck.yml
@@ -30,6 +30,20 @@
     - groups[nova_cell_compute_group] | length < 1
     - not nova_compute_ironic.enabled | bool
 
+- name: Checking nova multi-compute-ironic inventory
+  vars:
+    nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.assert:
+    msg: >
+      Each nova-compute-ironic-N group must contain a single host.
+    that:
+      - groups[nova_cell_compute_ironic_group ~ '-' ~ item] | length == 1
+  loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}"
+  run_once: true
+  when:
+    - nova_compute_ironic.iterate | bool
+    - nova_compute_ironic.enabled | bool
+
 - name: Checking free port for Nova NoVNC Proxy
   vars:
     service: "{{ nova_cell_services['nova-novncproxy'] }}"
diff --git a/ansible/roles/nova-cell/tasks/reload.yml b/ansible/roles/nova-cell/tasks/reload.yml
index 5724effdf..5f5c05b4e 100644
--- a/ansible/roles/nova-cell/tasks/reload.yml
+++ b/ansible/roles/nova-cell/tasks/reload.yml
@@ -37,4 +37,28 @@
   when:
     - kolla_action == 'upgrade'
     - service | service_enabled_and_mapped_to_host
+    - item != 'nova-compute-ironic' or not service.iterate | bool
   with_items: "{{ nova_cell_services_require_nova_conf }}"
+
+- name: Reload nova-compute-ironic to remove RPC version cap (iterated)
+  vars:
+    nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  become: true
+  kolla_container:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ nova_compute_ironic.container_name ~ '_' ~ item }}"
+    image: "{{ nova_compute_ironic.image }}"
+    environment: "{{ nova_compute_ironic.environment | default(omit) }}"
+    pid_mode: "{{ nova_compute_ironic.pid_mode | default('') }}"
+    ipc_mode: "{{ nova_compute_ironic.ipc_mode | default(omit) }}"
+    privileged: "{{ nova_compute_ironic.privileged | default(False) }}"
+    volumes: "{{ nova_compute_ironic.volumes | reject('equalto', '') | list }}"
+    dimensions: "{{ nova_compute_ironic.dimensions }}"
+    healthcheck: "{{ nova_compute_ironic.healthcheck | default(omit) }}"
+  when:
+    - kolla_action == 'upgrade'
+    - nova_compute_ironic | service_enabled
+    - nova_compute_ironic.iterate | bool
+    - inventory_hostname in groups[nova_compute_ironic.group ~ '-' ~ item]
+  loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}"
diff --git a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
index d930f8dea..eb7e80e83 100644
--- a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
+++ b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
@@ -13,13 +13,14 @@
       {{ virt_computes_in_batch |
          map('extract', hostvars, ['ansible_facts', 'nodename']) |
          list }}
-    # For ironic, use {{ansible_facts.hostname}}-ironic since this is what we
-    # configure for [DEFAULT] host in nova.conf.
     ironic_compute_service_hosts: >-
       {{ ironic_computes_in_batch |
-         map('extract', hostvars) | json_query('[].nova_compute_ironic_custom_host || [].ansible_facts.hostname') |
-         map('regex_replace', '^(.*)$', '\1-ironic') |
-         list }}
+         map('extract', hostvars) |
+         json_query(
+           '{classic: [].nova_compute_ironic_custom_host || [].ansible_facts.hostname,
+             multi: [].group_names}') |
+         get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }}
+
     expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}"
   block:
     - name: Waiting for nova-compute services to register themselves
@@ -73,13 +74,19 @@
         failed_compute_service_hosts: >-
           {{ expected_compute_service_hosts | difference(nova_compute_service_hosts) | list }}
         # Whether any compute services failed on this host.
+        ironic_compute_service_config_for_host:
+          classic: "{{ [nova_compute_ironic_custom_host | default(ansible_facts.hostname)] if inventory_hostname in ironic_computes_in_batch else [] }}"
+          multi: "{{ [group_names] }}"
+        expected_ironic_compute_services: >-
+          {{ ironic_compute_service_config_for_host | get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }}
         any_failed_services: >-
           {{ (ansible_facts.nodename | default()) in failed_compute_service_hosts or
-             ((ansible_facts.hostname | default()) ~ "-ironic") in failed_compute_service_hosts }}
+             expected_ironic_compute_services | intersect(failed_compute_service_hosts) | length > 0 }}
+
       ansible.builtin.fail:
         msg: >-
-          The Nova compute service failed to register itself on the following
-          hosts: {{ failed_compute_service_hosts | join(',') }}
+          The following Nova compute services failed to register
+          themselves: {{ failed_compute_service_hosts | join(',') }}
       when: >-
         any_failed_services or
         (nova_compute_registration_fatal | bool and
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index 2950dfcc4..7d30dcfd0 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -8,9 +8,27 @@ state_path = /var/lib/nova
 
 allow_resize_to_same_host = true
 
+{% set nova_compute_shard_key = nova_compute_shard_key | default("") %}
+{% set nova_compute_conductor_group = nova_compute_conductor_group | default("") %}
+{% set nova_compute_ironic_custom_host = nova_compute_ironic_custom_host | default("") %}
 {% if service_name == "nova-compute-ironic" %}
-host={{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+{%     if nova_compute_ironic_custom_host | length > 0 %}
+{#         NOTE(dougszu): This is provided for backwards compatibility #}
+{%         set host_value = nova_compute_ironic_custom_host ~ "-ironic" %}
+{%     elif nova_compute_conductor_group | length > 0 or nova_compute_shard_key | length > 0 %}
+{%         set host_value = (nova_compute_conductor_group ~ "-" ~ nova_compute_shard_key) ~ "-ironic" %}
+{%     else %}
+{#         NOTE(dougszu): This is to support legacy behaviour #}
+{%         set host_value = ansible_facts.hostname ~ "-ironic" %}
+{%     endif %}
+host = {{ host_value }}
+
+{%     if nova_compute_ironic_id is defined %}
+log_file = /var/log/kolla/nova/nova-compute-ironic-{{ nova_compute_ironic_id }}.log
+{%     else %}
 log_file = /var/log/kolla/nova/nova-compute-ironic.log
+{%     endif %}
+
 compute_driver = ironic.IronicDriver
 ram_allocation_ratio = 1.0
 reserved_host_memory_mb = 0
@@ -101,6 +119,19 @@ project_name = service
 user_domain_name = {{ default_user_domain_name }}
 project_domain_name = {{ default_project_domain_name }}
 endpoint_override = {{ ironic_internal_endpoint }}/v1
+
+{%     if nova_compute_shard_key is not in ['default', ''] %}
+shard = {{ nova_compute_shard_key }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in ['default', ''] %}
+conductor_group = {{ nova_compute_conductor_group }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in ['default', ''] and nova_compute_shard_key is in ['default', ''] %}
+{# NOTE(dougszu): In this case only, Nova Compute Ironic won't start unless the peer_list #}
+{# is populated. peer_list is a deprecated config option and should never have more than #}
+{# one host in it. This appears to be a bug. #}
+peer_list = {{ host_value }}
+{%     endif %}
 {% endif %}
 
 [oslo_concurrency]
@@ -180,7 +211,7 @@ driver = noop
 [oslo_messaging_rabbit]
 use_queue_manager = true
 {% if service_name == "nova-compute-ironic" %}
-hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+hostname = {{ host_value }}
 {% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
diff --git a/ansible/roles/service-cert-copy/tasks/iterated.yml b/ansible/roles/service-cert-copy/tasks/iterated.yml
new file mode 100644
index 000000000..4da5741fe
--- /dev/null
+++ b/ansible/roles/service-cert-copy/tasks/iterated.yml
@@ -0,0 +1,10 @@
+---
+- name: "Copying over extra CA certificates for {{ project_name }}"
+  become: true
+  ansible.builtin.copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ outer_item.key }}-{{ item }}/ca-certificates"
+    mode: "0644"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - inventory_hostname in groups[service.group ~ '-' ~ item]
diff --git a/ansible/roles/service-cert-copy/tasks/main.yml b/ansible/roles/service-cert-copy/tasks/main.yml
index d96bf0ca6..ef7ac6717 100644
--- a/ansible/roles/service-cert-copy/tasks/main.yml
+++ b/ansible/roles/service-cert-copy/tasks/main.yml
@@ -7,8 +7,21 @@
     mode: "0644"
   when:
     - kolla_copy_ca_into_containers | bool
+    - not (item.value.iterate | default(False)) | bool
   with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
 
+# NOTE(dougszu): For current iterable services, we only need CA certs
+- name: "Check containers that require iteration for {{ kolla_role_name | default(project_name) }}"
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ project_services | select_services_enabled_and_mapped_to_host | dict2items }}"
+  loop_control:
+    loop_var: outer_item
+  when:
+    - (service.iterate | default(False)) | bool
+    - kolla_copy_ca_into_containers | bool
+
 - name: "Copying over backend internal TLS certificate for {{ project_name }}"
   vars:
     certs:
diff --git a/ansible/roles/service-check-containers/tasks/iterated.yml b/ansible/roles/service-check-containers/tasks/iterated.yml
index bd266b115..c4f28e6d4 100644
--- a/ansible/roles/service-check-containers/tasks/iterated.yml
+++ b/ansible/roles/service-check-containers/tasks/iterated.yml
@@ -23,6 +23,12 @@
     command: "{{ service.command | default(omit) }}"
     cgroupns_mode: "{{ service.cgroupns_mode | default(omit) }}"
   loop: "{{ range(1, (iterate_count | int) + 1) | list }}"
+  when:
+    # NOTE(dougszu): Due to issues with nova-compute-ironic HA, we can't have
+    # more than one instance for a given 'item'. Eg. nova_compute_ironic_1
+    # should never run on more than one host.
+    # See: https://specs.openstack.org/openstack/nova-specs/specs/2024.1/implemented/ironic-shards.html
+    - service_name != 'nova-compute-ironic' or inventory_hostname in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
   register: container_check
 
 - name: "Store changed containers to pass to the handler"
diff --git a/ansible/roles/service-precheck/tasks/iterated.yml b/ansible/roles/service-precheck/tasks/iterated.yml
new file mode 100644
index 000000000..d9157ddce
--- /dev/null
+++ b/ansible/roles/service-precheck/tasks/iterated.yml
@@ -0,0 +1,13 @@
+---
+- name: "Validate inventory groups (iterated) for {{ project_name }}"
+  vars:
+    service_name: "{{ outer_item.key }}"
+    service: "{{ outer_item.value }}"
+    service_group: "{{ service.group ~ '-' ~ item }}"
+  ansible.builtin.fail:
+    msg: >-
+      Ansible inventory does not contain the expected group {{ service_group }}
+      for service {{ service_name }} in {{ project_name }}.
+  when:
+    - service_group not in groups
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
diff --git a/ansible/roles/service-precheck/tasks/main.yml b/ansible/roles/service-precheck/tasks/main.yml
index 58448555e..81b90f6a5 100644
--- a/ansible/roles/service-precheck/tasks/main.yml
+++ b/ansible/roles/service-precheck/tasks/main.yml
@@ -9,7 +9,19 @@
       for service {{ service_name }} in {{ project_name }}.
   loop: "{{ query('dict', service_precheck_services) }}"
   when:
+    - not (service.iterate | default(False)) | bool
     - "'group' in service"
     - service.group not in groups
   loop_control:
     label: "{{ service_name }}"
+
+- name: Include tasks
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ query('dict', service_precheck_services) }}"
+  loop_control:
+    loop_var: outer_item
+  when:
+    - (service.iterate | default(False)) | bool
+    - "'group' in service"
diff --git a/ansible/roles/service-stop/tasks/iterated.yml b/ansible/roles/service-stop/tasks/iterated.yml
index f25bf0c26..cf4e8e489 100644
--- a/ansible/roles/service-stop/tasks/iterated.yml
+++ b/ansible/roles/service-stop/tasks/iterated.yml
@@ -11,4 +11,5 @@
     ignore_missing: "{{ kolla_action_stop_ignore_missing | bool }}"
   when:
     - service.container_name not in skip_stop_containers
+    - service_name != 'nova-compute-ironic' or inventory_hostname in groups[service.group ~ '-' ~ item]
   loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
diff --git a/doc/source/admin/advanced-configuration.rst b/doc/source/admin/advanced-configuration.rst
index 7ece68e3a..c85a498dc 100644
--- a/doc/source/admin/advanced-configuration.rst
+++ b/doc/source/admin/advanced-configuration.rst
@@ -178,6 +178,113 @@ operator needs to create ``/etc/kolla/config/global.conf`` with content:
    [database]
    max_pool_size = 100
 
+Large baremetal deployments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Out of the box, a typical Kolla Ansible deployment can support managing a few
+hundred baremetal nodes. Beyond this number, it becomes necessary to configure
+scaling mechanisms built into Nova and Ironic.
+
+There are two mechanisms to consider:
+  - shards
+  - conductor groups
+
+Please see the Ironic documentation for further `information
+<https://docs.openstack.org/ironic/latest/admin/availability-zones.html#common-deployment-patterns>`_.
+
+As an example, consider a deployment of 700 baremetal nodes spread over two
+distinct locations. Location 1 consists of 500 baremetal nodes, and location 2
+consists of 200 baremetal nodes.
+
+For location 1, all 500 nodes are placed into the same Ironic conductor
+group. Ironic conductors configured to use this group are deployed on each
+of the three nodes in the control plane. A single Nova Compute Ironic service
+would struggle to manage this many nodes, so two shards are created. Due to
+HA limitations with Nova Compute Ironic, a single instance is created for
+each shard, and the instances may be deployed to any node in the control
+plane.
+
+For location 2, no shards are required due to the smaller number of baremetal
+nodes. A single conductor group is configured. The result is that three Ironic
+conductors are deployed (one on each node in the control plane), and a single
+instance of Nova Compute Ironic configured to use this conductor group.
+Furthermore, for this location ``custom_host`` is used to override the
+automatically generated host field in the configuration. This is useful for
+migrating to the multi-instance configuration described here.
+
+Finally, Ironic and Nova services are deployed with no configured shards or
+conductor groups as a catch all.
+
+.. code-block:: yaml
+
+  nova_multi_compute_ironic_config:
+    - "shard_key": "shard_1"
+      "conductor_group": "location_1"
+    - "shard_key": "shard_2"
+      "conductor_group": "location_1"
+    - "conductor_group": "location_2"
+      "custom_host": "some_custom_host_field"
+    - {}
+
+.. note::
+
+   ``nova_multi_compute_ironic_config`` must be defined as a global variable
+   and not customised via hostvars.
+
+.. note::
+
+   If you currently have the ``nova-compute-ironic`` service deployed,
+   you must manually remove it from the controllers before migrating to the
+   new mode. You will need to ensure that you configure a compatible service
+   in ``nova_multi_compute_ironic_config`` to replace the old one. This will
+   likely involve setting the ``custom_host`` field at the very minimum.
+
+The placement of the above services is managed via the inventory. This
+allows flexible placement, should a controller fail.
+
+.. code-block:: yaml
+
+  [nova-compute-ironic-1]
+  controller-01
+  [nova-compute-ironic-2]
+  controller-02
+  [nova-compute-ironic-3]
+  controller-03
+  [nova-compute-ironic-4]
+  controller-03
+
+  [nova-compute-ironic:children]
+  nova-compute-ironic-1
+  nova-compute-ironic-2
+  nova-compute-ironic-3
+  nova-compute-ironic-4
+
+.. note::
+
+   The number of ``nova-compute-ironic-N`` groups must equal the length of
+   ``nova_multi_compute_ironic_config``. This variable is an ordered list,
+   where the first item maps to ``nova-compute-ironic-1``, the second item
+   to ``nova-compute-ironic-2`` and so forth.
+
+.. note::
+
+   A 'cleanup' function is provided which assists in 'failing over'
+   service instances to other hosts. For example, if ``controller-01``
+   was to be taken down for maintenance, replacing it with ``controller-02``
+   in the ``nova-compute-ironic-1`` group and re-deploying the ``nova``
+   service would result in the service being removed from ``controller-01``
+   and re-deployed on ``controller-02``. The cleanup function cannot
+   currently handle downsizing the number of service instances. In this case
+   the redundant service instances must be cleaned up manually. For obvious
+   reasons the cleanup function cannot remove a service from a failed node,
+   and in such a case, you must be careful not to allow a second identical
+   service instance to start when the failed node is recovered.
+
+The ``nova-compute-ironic`` service instances may be configured
+individually via the existing override mechanism. For example, the
+creation of ``nova/nova-compute-ironic-1.conf`` will allow variables
+for that specific service to be overridden.
+
 OpenStack policy customisation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/kolla_ansible/nova_filters.py b/kolla_ansible/nova_filters.py
index 4bb5cbdf6..f9be3eb98 100644
--- a/kolla_ansible/nova_filters.py
+++ b/kolla_ansible/nova_filters.py
@@ -15,6 +15,8 @@
 import jinja2
 import re
 
+from kolla_ansible import exception
+
 
 def extract_cell(list_cells_cli_output, cell_name):
     """Extract cell settings from nova_manage CLI
@@ -85,8 +87,57 @@ def namespace_haproxy_for_cell(services, cell_name):
     return services
 
 
+def get_expected_ironic_compute_services(ironic_compute_batch_info,
+                                         multi_compute_conf,
+                                         nova_cell_compute_ironic_group):
+    """Return a List of expected Ironic compute services
+
+      :param ironic_compute_batch_info: A dict containing configuration
+             information about Ironic Compute hosts.
+      :param multi_compute_conf: Ironic multi-compute config
+      :param nova_cell_compute_ironic_group: Ansible group name for
+             nova-compute-ironic. Eg. 'nova-compute-ironic'.
+      :returns: A list of Nova Compute Ironic service hostnames.
+    """
+    classic_info = ironic_compute_batch_info['classic']
+    multi_compute_info = ironic_compute_batch_info['multi']
+    if len(multi_compute_conf) == 0:
+        return [f'{host}-ironic' for host in classic_info]
+
+    expected_services = []
+    group_prefix = nova_cell_compute_ironic_group + '-'
+    for host in multi_compute_info:
+        iterate_vars = [
+            int(group.split(group_prefix)[1])
+            for group in host if group_prefix in group]
+        for i in iterate_vars:
+            try:
+                service_conf = multi_compute_conf[i - 1]
+            except IndexError:
+                raise exception.FilterError(
+                    "Unable to look up multi-compute Ironic config for "
+                    "{g}-{i} inventory group. Check that "
+                    "nova_multi_compute_ironic_config has an entry for "
+                    "each defined group.".format(
+                        g=nova_cell_compute_ironic_group,
+                        i=i))
+            expected_services.append(
+                _get_expected_compute_service_name(service_conf))
+    return expected_services
+
+
+def _get_expected_compute_service_name(service_conf):
+    custom_host = service_conf.get('custom_host')
+    if custom_host:
+        return f"{custom_host}-ironic"
+    cg = service_conf.get('conductor_group', 'default')
+    sk = service_conf.get('shard_key', 'default')
+    return f"{cg}-{sk}-ironic"
+
+
 def get_filters():
     return {
         "extract_cell": extract_cell,
         "namespace_haproxy_for_cell": namespace_haproxy_for_cell,
+        "get_expected_ironic_compute_services": get_expected_ironic_compute_services,  # noqa
     }
diff --git a/kolla_ansible/tests/unit/test_nova_filters.py b/kolla_ansible/tests/unit/test_nova_filters.py
index e8d5c507a..d436b0ef1 100644
--- a/kolla_ansible/tests/unit/test_nova_filters.py
+++ b/kolla_ansible/tests/unit/test_nova_filters.py
@@ -73,6 +73,92 @@ class TestFilters(unittest.TestCase):
         self.assertRaisesRegex(jinja2.TemplateRuntimeError, 'duplicates',
                                self._test_extract_cell, test_data, 'cell0001')
 
+    def test_get_expected_ironic_compute_services_multi_compute(self):
+        example_ironic_compute_conf = {
+            # We expect the classic config to be ignored
+            'classic': ['custom-host-nova-compute'],
+            # We have a batch of 2 Ironic compute hosts which should
+            # run a total of 3 Nova compute Ironic instances
+            'multi': [
+                ['nova-compute-ironic-1',
+                 'nova-compute-ironic-2',
+                 'nova-compute-ironic',
+                 'something'],
+                ['nova-compute-ironic-3',
+                    'nova-compute-ironic',
+                    'something'],
+            ]
+        }
+        multi_conf = [
+            {"custom_host": "some_custom_host"},
+            {"shard_key": "shard_1", "conductor_group": "location_1"},
+            {"shard_key": "shard_2", "conductor_group": "location_1"},
+            {"conductor_group": "location_2"},
+            {"shard_key": "shard_1"}
+        ]
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, multi_conf, 'nova-compute-ironic')
+        expected = [
+            'some_custom_host-ironic',
+            'location_1-shard_1-ironic',
+            'location_1-shard_2-ironic',
+        ]
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_no_compute(self):
+        example_ironic_compute_conf = {
+            'classic': [],
+            'multi': []
+        }
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, [], 'nova-compute-ironic')
+        expected = []
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_classic_compute(self):
+        example_ironic_compute_conf = {
+            'classic': ['custom-foo'],
+            'multi': []
+        }
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, [], 'nova-compute-ironic')
+        expected = ['custom-foo-ironic']
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_classic_compute_multi(self):
+        # NOTE(dougszu): This isn't a recommended configuration
+        example_ironic_compute_conf = {
+            'classic': ['custom-foo', 'ctrl02', 'ctrl03'],
+            'multi': []
+        }
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, [], 'nova-compute-ironic')
+        expected = [
+            'custom-foo-ironic',
+            'ctrl02-ironic',
+            'ctrl03-ironic',
+        ]
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_bad_config(self):
+        example_ironic_compute_conf = {
+            'classic': [],
+            'multi': [
+                ['my-custom-group-1',
+                 'my-custom-group-2',
+                 'my-custom-group',
+                 'something'],
+            ]
+        }
+        multi_conf = [
+            {"shard_key": "shard_1", "conductor_group": "location_1"},
+        ]
+        with self.assertRaisesRegex(Exception,
+                                    'Unable to look up multi-compute Ironic '
+                                    'config for my-custom-group-2'):
+            filters.get_expected_ironic_compute_services(
+                example_ironic_compute_conf, multi_conf, 'my-custom-group')
+
     def test_namespace_haproxy_for_cell_with_empty_name(self):
         example_services = {
             'nova-novncproxy': {
diff --git a/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml
new file mode 100644
index 000000000..c99523feb
--- /dev/null
+++ b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds support for deploying multiple instances of the Nova Compute Ironic
+    service on the same host. This is useful in large baremetal deployments.
+upgrade:
+  - |
+    If you are migrating to the new feature to deploy multiple instances of
+    Nova Compute Ironic, you are first required to manually remove the
+    existing Nova Compute Ironic Docker container, and config directories.
diff --git a/roles/openstack-clients/defaults/main.yml b/roles/openstack-clients/defaults/main.yml
index e1f53e279..034618843 100644
--- a/roles/openstack-clients/defaults/main.yml
+++ b/roles/openstack-clients/defaults/main.yml
@@ -11,7 +11,7 @@ openstack_clients_pip_packages:
   - package: python-heatclient
     enabled: true
   - package: python-ironicclient
-    enabled: "{{ scenario == 'ironic' }}"
+    enabled: "{{ scenario in ['ironic', 'multi-compute-ironic'] }}"
   - package: python-magnumclient
     enabled: "{{ scenario == 'magnum' }}"
   - package: python-neutronclient
diff --git a/tests/run.yml b/tests/run.yml
index 8dd6c62c6..efa817e24 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -208,10 +208,10 @@
         # ironic.conf
         - src: "tests/templates/ironic-overrides.j2"
           dest: /etc/kolla/config/ironic.conf
-          when: "{{ scenario == 'ironic' }}"
+          when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}"
         - src: "tests/templates/tenks-deploy-config.yml.j2"
           dest: "{{ ansible_env.HOME }}/tenks.yml"
-          when: "{{ scenario == 'ironic' }}"
+          when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}"
         # keystone-federation config
         - src: "tests/templates/keystone-federation/idp.example.org%2Frealms%2Fexample.conf.j2"
           dest: "/etc/kolla/config/keystone-httpd/federation/oidc/metadata/idp.example.org%2Frealms%2Fexample.conf"
@@ -243,7 +243,7 @@
               dest: ironic-agent.initramfs
             - src: "ipa-centos9-{{ zuul.branch | replace('/', '-') }}.kernel"
               dest: ironic-agent.kernel
-      when: scenario == "ironic"
+      when: scenario == "ironic" or scenario == "multi-compute-ironic"
 
     - block:
         - name: Slurp requirements.yml
@@ -427,7 +427,7 @@
             EXT_NET_GATEWAY: "{{ neutron_external_network_prefix }}1"
             EXT_NET_DEMO_ROUTER_ADDR: "{{ neutron_external_network_prefix }}10"
             SCENARIO: "{{ scenario }}"
-          when: openstack_core_tested or scenario in ['ironic', 'magnum', 'nfv', 'zun', 'octavia']
+          when: openstack_core_tested or scenario in ['ironic', 'multi-compute-ironic' ,'magnum', 'nfv', 'zun', 'octavia']
 
         - name: Test OVN VPNaaS
           import_role:
@@ -494,7 +494,8 @@
                 chdir: "{{ kolla_ansible_src_dir }}"
               environment:
                 TLS_ENABLED: "{{ tls_enabled }}"
-          when: scenario == "ironic"
+                MULTI_COMPUTE_IRONIC: "{{ scenario == 'multi-compute-ironic' }}"
+          when: scenario == "ironic" or scenario == "multi-compute-ironic"
 
         - name: Run test-magnum.sh script
           script:
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index f5f005acd..ca2ad562e 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -138,6 +138,22 @@ enable_proxysql: "yes"
 enable_prometheus_proxysql_exporter: "yes"
 {% endif %}
 
+{% if scenario == "multi-compute-ironic" %}
+nova_multi_compute_ironic_config:
+  - {}
+  - "shard_key": "shard_1"
+    "conductor_group": "location_1"
+  - "shard_key": "shard_2"
+    "conductor_group": "location_1"
+  - "conductor_group": "location_2"
+
+enable_ironic: "yes"
+enable_ironic_pxe_filter: "yes"
+ironic_dnsmasq_dhcp_ranges:
+  - range: "10.42.0.2,10.42.0.254,255.255.255.0"
+ironic_internal_endpoint: http://192.0.2.1:6385
+{% endif %}
+
 {% if scenario == "mariadb" %}
 enable_fluentd: "yes"
 enable_mariadb: "yes"
diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2
index 2991518ce..a3bea7213 100644
--- a/tests/templates/inventory.j2
+++ b/tests/templates/inventory.j2
@@ -6,7 +6,7 @@
 # primary and secondary while the parent group (control) uses only primary.
 
 [control]
-{% if scenario == 'masakari' %}
+{% if scenario in ['masakari', 'multi-compute-ironic'] %}
 {%     for host in hostvars if host in ['primary'] %}
 {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }}
 {%     endfor %}
@@ -299,8 +299,25 @@ nova
 [nova-spicehtml5proxy:children]
 nova
 
+{% if scenario == 'multi-compute-ironic' %}
+[nova-compute-ironic-1]
+primary
+[nova-compute-ironic-2]
+secondary1
+[nova-compute-ironic-3]
+secondary2
+[nova-compute-ironic-4]
+secondary2
+
+[nova-compute-ironic:children]
+nova-compute-ironic-1
+nova-compute-ironic-2
+nova-compute-ironic-3
+nova-compute-ironic-4
+{% else %}
 [nova-compute-ironic:children]
 nova
+{% endif %}
 
 [nova-serialproxy:children]
 nova
diff --git a/tests/templates/ironic-overrides.j2 b/tests/templates/ironic-overrides.j2
index be944b307..9603d6b2c 100644
--- a/tests/templates/ironic-overrides.j2
+++ b/tests/templates/ironic-overrides.j2
@@ -2,6 +2,16 @@
 enabled_inspect_interfaces = no-inspect, agent
 default_inspect_interface = agent
 
+{% if scenario == "multi-compute-ironic" %}
+{%     raw %}
+{%     set cg = 'location_1' if inventory_hostname == 'secondary1' else 'location_2' if inventory_hostname == 'secondary2' %}
+{%     if cg %}
+[conductor]
+conductor_group = {{ cg }}
+{%     endif %}
+{%     endraw %}
+{% endif %}
+
 [neutron]
 cleaning_network = public1
 inspection_network = public1
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 0758a8067..7cb2ec76f 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -4,6 +4,7 @@
     templates:
       - ansible-role-jobs
       - check-requirements
+      - kolla-ansible-scenario-multi-compute-ironic
       - kolla-ansible-scenario-aio
       - kolla-ansible-scenario-bifrost
       - kolla-ansible-scenario-cells
diff --git a/zuul.d/scenarios/multi-compute-ironic.yaml b/zuul.d/scenarios/multi-compute-ironic.yaml
new file mode 100644
index 000000000..814116a1f
--- /dev/null
+++ b/zuul.d/scenarios/multi-compute-ironic.yaml
@@ -0,0 +1,47 @@
+---
+- job:
+    name: kolla-ansible-multi-compute-ironic-base
+    parent: kolla-ansible-base
+    voting: false
+    files: !inherit
+      - ^ansible/group_vars/all/(nova|ironic).yml
+      - ^ansible/roles/(nova|nova-cell|ironic)/
+      - ^tests/deploy-tenks\.sh$
+      - ^tests/templates/ironic-overrides\.j2$
+      - ^tests/templates/tenks-deploy-config\.yml\.j2$
+      - ^tests/test-dashboard\.sh$
+      - ^tests/test-ironic\.sh$
+    required-projects:
+      - openstack/tenks
+    vars:
+      scenario: multi-compute-ironic
+      scenario_images_extra:
+        - ^dnsmasq
+        - ^ironic
+        - ^iscsid
+      tls_enabled: false
+
+- job:
+    name: kolla-ansible-debian-trixie-multi-compute-ironic
+    parent: kolla-ansible-multi-compute-ironic-base
+    nodeset: kolla-ansible-debian-trixie-multi-16GB
+
+- job:
+    name: kolla-ansible-rocky-10-multi-compute-ironic
+    parent: kolla-ansible-multi-compute-ironic-base
+    nodeset: kolla-ansible-rocky-10-multi-16GB
+
+- job:
+    name: kolla-ansible-ubuntu-noble-multi-compute-ironic
+    parent: kolla-ansible-multi-compute-ironic-base
+    nodeset: kolla-ansible-ubuntu-noble-multi-16GB
+
+- project-template:
+    name: kolla-ansible-scenario-multi-compute-ironic
+    description: |
+      Runs Kolla-Ansible Nova Multi Compute Ironic scenario jobs.
+    check:
+      jobs:
+        - kolla-ansible-debian-trixie-multi-compute-ironic
+        - kolla-ansible-rocky-10-multi-compute-ironic
+        - kolla-ansible-ubuntu-noble-multi-compute-ironic