Merge "Support multiple Nova Compute Ironic instances"

27 files changed, 627 insertions, 19 deletions

diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml
index 6979408e9..790190fb0 100644
--- a/ansible/roles/nova-cell/defaults/main.yml
+++ b/ansible/roles/nova-cell/defaults/main.yml
@@ -72,6 +72,8 @@ nova_cell_services:
     volumes: "{{ nova_compute_ironic_default_volumes + nova_compute_ironic_extra_volumes }}"
     dimensions: "{{ nova_compute_ironic_dimensions }}"
     healthcheck: "{{ nova_compute_ironic_healthcheck }}"
+    iterate: "{{ nova_multi_compute_ironic_config | length > 0 }}"
+    iterate_var: "{{ nova_multi_compute_ironic_config | length }}"
 
 ####################
 # Config Validate
@@ -501,7 +503,7 @@ nova_compute_default_volumes:
   - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
   - "{{ kolla_dev_repos_directory ~ '/nova:/dev-mode/nova' if nova_dev_mode | bool else '' }}"
 nova_compute_ironic_default_volumes:
-  - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
+  - "{{ node_config_directory }}/nova-compute-ironic{{ '-' ~ item if item|default(0)|int > 0 else '' }}/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "kolla_logs:/var/log/kolla/"
   - "{{ '/dev/shm:/dev/shm' }}"
@@ -675,6 +677,17 @@ nova_libvirt_cleanup_remove_volumes: false
 # Database
 nova_cell_database_enable_tls_internal: "{{ database_enable_tls_internal | bool }}"
 
+##############################
+# Nova Compute Ironic scaling
+##############################
+
+# The following option can be used to deploy multiple instances of Nova
+# Compute Ironic per host. This is useful for deployments with a large
+# number of baremetal nodes (>200). Please see the advanced documentation
+# section for further details.
+
+nova_multi_compute_ironic_config: []
+
 ###################
 # Copy certificates
 ###################
diff --git a/ansible/roles/nova-cell/handlers/main.yml b/ansible/roles/nova-cell/handlers/main.yml
index a67a1231f..c6ac28da9 100644
--- a/ansible/roles/nova-cell/handlers/main.yml
+++ b/ansible/roles/nova-cell/handlers/main.yml
@@ -162,16 +162,21 @@
   vars:
     service_name: "nova-compute-ironic"
     service: "{{ nova_cell_services[service_name] }}"
+    item: "{{ changed_container.item }}"
+    container_name_postfix: "{{ '' if changed_container == 'legacy' else '_' ~ item }}"
   become: true
   kolla_container:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
+    name: "{{ service.container_name + container_name_postfix }}"
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes | reject('equalto', '') | list }}"
     dimensions: "{{ service.dimensions }}"
     healthcheck: "{{ service.healthcheck | default(omit) }}"
+  loop_control:
+    loop_var: changed_container
+  loop: "{{ nova_compute_ironic_changed_containers | default(['legacy']) }}"
 
 # nova-compute-fake is special. It will start multi numbers of container
 # so put all variables here rather than defaults/main.yml file
diff --git a/ansible/roles/nova-cell/tasks/cleanup.yml b/ansible/roles/nova-cell/tasks/cleanup.yml
new file mode 100644
index 000000000..0f0952823
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/cleanup.yml
@@ -0,0 +1,26 @@
+---
+- name: Stop and remove containers for unmanaged services
+  become: true
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  kolla_container:
+    action: "stop_and_remove_container"
+    name: "{{ service.container_name ~ '_' ~ item }}"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
+
+- name: Removing config for unmanaged services
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/{{ service_name ~ '-' ~ item }}"
+    state: "absent"
+  become: true
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
diff --git a/ansible/roles/nova-cell/tasks/config-compute-ironic.yml b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml
new file mode 100644
index 000000000..0f1c05a9f
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml
@@ -0,0 +1,67 @@
+---
+- name: "Ensuring config directories exist for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+
+- name: "Copying over config.json files for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.template:
+    src: "nova-compute-ironic.json.j2"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/config.json"
+    mode: "0660"
+  become: true
+
+- name: "Get Nova Compute Ironic multi-instance configuration for instance {{ item }}"
+  ansible.builtin.set_fact:
+    nova_multi_compute_ironic_instance_config: "{{ nova_multi_compute_ironic_config[(item | int) - 1] }}"
+
+- name: "Generate config files for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+    nova_compute_shard_key: "{{ nova_multi_compute_ironic_instance_config['shard_key'] | default('default') }}"
+    nova_compute_conductor_group: "{{ nova_multi_compute_ironic_instance_config['conductor_group'] | default('default') }}"
+    nova_compute_ironic_custom_host: "{{ nova_multi_compute_ironic_instance_config['custom_host'] | default('') }}"
+    nova_compute_ironic_id: "{{ item }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/nova.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}-{{ item }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}-{{ item }}.conf"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/nova.conf"
+    mode: "0660"
+  become: true
+
+- name: "Copying over existing policy file for item {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.template:
+    src: "{{ nova_policy_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/{{ nova_policy_file }}"
+    mode: "0660"
+  become: true
+  when:
+    - nova_policy_file is defined
+
+- name: "Copying over vendordata file for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.copy:
+    src: "{{ vendordata_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/vendordata.json"
+    mode: "0660"
+  become: true
+  when:
+    - vendordata_file_path is defined
diff --git a/ansible/roles/nova-cell/tasks/config.yml b/ansible/roles/nova-cell/tasks/config.yml
index 87104f89e..7f0cfca22 100644
--- a/ansible/roles/nova-cell/tasks/config.yml
+++ b/ansible/roles/nova-cell/tasks/config.yml
@@ -7,6 +7,8 @@
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0770"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over TLS certificates
@@ -58,6 +60,8 @@
     src: "{{ item.key }}.json.j2"
     dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
     mode: "0660"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over nova.conf
@@ -76,8 +80,18 @@
     mode: "0660"
   when:
     - item.key in nova_cell_services_require_nova_conf
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
+- name: Copying over config for Nova compute Ironic multi-instance
+  vars:
+    service: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.include_tasks: config-compute-ironic.yml
+  loop: "{{ range(1, nova_multi_compute_ironic_config | length + 1) | list }}"
+  when:
+    - nova_multi_compute_ironic_config | length > 0
+    - inventory_hostname in groups[service.group ~ '-' ~ item]
+
 - name: Copying over Nova compute provider config
   become: true
   vars:
@@ -196,6 +210,7 @@
   when:
     - nova_policy_file is defined
     - item.key in nova_cell_services_require_policy_json
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over vendordata file to containers
@@ -209,6 +224,7 @@
   when:
     - vendordata_file_path is defined
     - service | service_enabled_and_mapped_to_host
+    - item != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_items:
     - nova-compute
     - nova-compute-ironic
diff --git a/ansible/roles/nova-cell/tasks/deploy.yml b/ansible/roles/nova-cell/tasks/deploy.yml
index 2a416d8d4..288531557 100644
--- a/ansible/roles/nova-cell/tasks/deploy.yml
+++ b/ansible/roles/nova-cell/tasks/deploy.yml
@@ -6,6 +6,10 @@
 - name: Import tasks from version-check.yml
   ansible.builtin.import_tasks: version-check.yml
 
+# NOTE(dougszu): Currently only covers nova-compute-ironic
+- name: Cleanup stale configuration
+  ansible.builtin.import_tasks: cleanup.yml
+
 - name: Import tasks from config-host.yml
   ansible.builtin.import_tasks: config-host.yml
 
diff --git a/ansible/roles/nova-cell/tasks/precheck.yml b/ansible/roles/nova-cell/tasks/precheck.yml
index a509e790c..9364c619e 100644
--- a/ansible/roles/nova-cell/tasks/precheck.yml
+++ b/ansible/roles/nova-cell/tasks/precheck.yml
@@ -30,6 +30,20 @@
     - groups[nova_cell_compute_group] | length < 1
     - not nova_compute_ironic.enabled | bool
 
+- name: Checking nova multi-compute-ironic inventory
+  vars:
+    nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.assert:
+    msg: >
+      Each nova-compute-ironic-N group must contain a single host.
+    that:
+      - groups[nova_cell_compute_ironic_group ~ '-' ~ item] | length == 1
+  loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}"
+  run_once: true
+  when:
+    - nova_compute_ironic.iterate | bool
+    - nova_compute_ironic.enabled | bool
+
 - name: Checking free port for Nova NoVNC Proxy
   vars:
     service: "{{ nova_cell_services['nova-novncproxy'] }}"
diff --git a/ansible/roles/nova-cell/tasks/reload.yml b/ansible/roles/nova-cell/tasks/reload.yml
index 5724effdf..5f5c05b4e 100644
--- a/ansible/roles/nova-cell/tasks/reload.yml
+++ b/ansible/roles/nova-cell/tasks/reload.yml
@@ -37,4 +37,28 @@
   when:
     - kolla_action == 'upgrade'
     - service | service_enabled_and_mapped_to_host
+    - item != 'nova-compute-ironic' or not service.iterate | bool
   with_items: "{{ nova_cell_services_require_nova_conf }}"
+
+- name: Reload nova-compute-ironic to remove RPC version cap (iterated)
+  vars:
+    nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  become: true
+  kolla_container:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ nova_compute_ironic.container_name ~ '_' ~ item }}"
+    image: "{{ nova_compute_ironic.image }}"
+    environment: "{{ nova_compute_ironic.environment | default(omit) }}"
+    pid_mode: "{{ nova_compute_ironic.pid_mode | default('') }}"
+    ipc_mode: "{{ nova_compute_ironic.ipc_mode | default(omit) }}"
+    privileged: "{{ nova_compute_ironic.privileged | default(False) }}"
+    volumes: "{{ nova_compute_ironic.volumes | reject('equalto', '') | list }}"
+    dimensions: "{{ nova_compute_ironic.dimensions }}"
+    healthcheck: "{{ nova_compute_ironic.healthcheck | default(omit) }}"
+  when:
+    - kolla_action == 'upgrade'
+    - nova_compute_ironic | service_enabled
+    - nova_compute_ironic.iterate | bool
+    - inventory_hostname in groups[nova_compute_ironic.group ~ '-' ~ item]
+  loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}"
diff --git a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
index d930f8dea..eb7e80e83 100644
--- a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
+++ b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
@@ -13,13 +13,14 @@
       {{ virt_computes_in_batch |
          map('extract', hostvars, ['ansible_facts', 'nodename']) |
          list }}
-    # For ironic, use {{ansible_facts.hostname}}-ironic since this is what we
-    # configure for [DEFAULT] host in nova.conf.
     ironic_compute_service_hosts: >-
       {{ ironic_computes_in_batch |
-         map('extract', hostvars) | json_query('[].nova_compute_ironic_custom_host || [].ansible_facts.hostname') |
-         map('regex_replace', '^(.*)$', '\1-ironic') |
-         list }}
+         map('extract', hostvars) |
+         json_query(
+           '{classic: [].nova_compute_ironic_custom_host || [].ansible_facts.hostname,
+             multi: [].group_names}') |
+         get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }}
+
     expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}"
   block:
     - name: Waiting for nova-compute services to register themselves
@@ -73,13 +74,19 @@
         failed_compute_service_hosts: >-
           {{ expected_compute_service_hosts | difference(nova_compute_service_hosts) | list }}
         # Whether any compute services failed on this host.
+        ironic_compute_service_config_for_host:
+          classic: "{{ [nova_compute_ironic_custom_host | default(ansible_facts.hostname)] if inventory_hostname in ironic_computes_in_batch else [] }}"
+          multi: "{{ [group_names] }}"
+        expected_ironic_compute_services: >-
+          {{ ironic_compute_service_config_for_host | get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }}
         any_failed_services: >-
           {{ (ansible_facts.nodename | default()) in failed_compute_service_hosts or
-             ((ansible_facts.hostname | default()) ~ "-ironic") in failed_compute_service_hosts }}
+             expected_ironic_compute_services | intersect(failed_compute_service_hosts) | length > 0 }}
+
       ansible.builtin.fail:
         msg: >-
-          The Nova compute service failed to register itself on the following
-          hosts: {{ failed_compute_service_hosts | join(',') }}
+          The following Nova compute services failed to register
+          themselves: {{ failed_compute_service_hosts | join(',') }}
       when: >-
         any_failed_services or
         (nova_compute_registration_fatal | bool and
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index 2950dfcc4..7d30dcfd0 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -8,9 +8,27 @@ state_path = /var/lib/nova
 
 allow_resize_to_same_host = true
 
+{% set nova_compute_shard_key = nova_compute_shard_key | default("") %}
+{% set nova_compute_conductor_group = nova_compute_conductor_group | default("") %}
+{% set nova_compute_ironic_custom_host = nova_compute_ironic_custom_host | default("") %}
 {% if service_name == "nova-compute-ironic" %}
-host={{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+{%     if nova_compute_ironic_custom_host | length > 0 %}
+{#         NOTE(dougszu): This is provided for backwards compatibility #}
+{%         set host_value = nova_compute_ironic_custom_host ~ "-ironic" %}
+{%     elif nova_compute_conductor_group | length > 0 or nova_compute_shard_key | length > 0 %}
+{%         set host_value = (nova_compute_conductor_group ~ "-" ~ nova_compute_shard_key) ~ "-ironic" %}
+{%     else %}
+{#         NOTE(dougszu): This is to support legacy behaviour #}
+{%         set host_value = ansible_facts.hostname ~ "-ironic" %}
+{%     endif %}
+host = {{ host_value }}
+
+{%     if nova_compute_ironic_id is defined %}
+log_file = /var/log/kolla/nova/nova-compute-ironic-{{ nova_compute_ironic_id }}.log
+{%     else %}
 log_file = /var/log/kolla/nova/nova-compute-ironic.log
+{%     endif %}
+
 compute_driver = ironic.IronicDriver
 ram_allocation_ratio = 1.0
 reserved_host_memory_mb = 0
@@ -101,6 +119,19 @@ project_name = service
 user_domain_name = {{ default_user_domain_name }}
 project_domain_name = {{ default_project_domain_name }}
 endpoint_override = {{ ironic_internal_endpoint }}/v1
+
+{%     if nova_compute_shard_key is not in ['default', ''] %}
+shard = {{ nova_compute_shard_key }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in ['default', ''] %}
+conductor_group = {{ nova_compute_conductor_group }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in ['default', ''] and nova_compute_shard_key is in ['default', ''] %}
+{# NOTE(dougszu): In this case only, Nova Compute Ironic won't start unless the peer_list #}
+{# is populated. peer_list is a deprecated config option and should never have more than #}
+{# one host in it. This appears to be a bug. #}
+peer_list = {{ host_value }}
+{%     endif %}
 {% endif %}
 
 [oslo_concurrency]
@@ -180,7 +211,7 @@ driver = noop
 [oslo_messaging_rabbit]
 use_queue_manager = true
 {% if service_name == "nova-compute-ironic" %}
-hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+hostname = {{ host_value }}
 {% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}
diff --git a/ansible/roles/service-cert-copy/tasks/iterated.yml b/ansible/roles/service-cert-copy/tasks/iterated.yml
new file mode 100644
index 000000000..4da5741fe
--- /dev/null
+++ b/ansible/roles/service-cert-copy/tasks/iterated.yml
@@ -0,0 +1,10 @@
+---
+- name: "Copying over extra CA certificates for {{ project_name }}"
+  become: true
+  ansible.builtin.copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ outer_item.key }}-{{ item }}/ca-certificates"
+    mode: "0644"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - inventory_hostname in groups[service.group ~ '-' ~ item]
diff --git a/ansible/roles/service-cert-copy/tasks/main.yml b/ansible/roles/service-cert-copy/tasks/main.yml
index d96bf0ca6..ef7ac6717 100644
--- a/ansible/roles/service-cert-copy/tasks/main.yml
+++ b/ansible/roles/service-cert-copy/tasks/main.yml
@@ -7,8 +7,21 @@
     mode: "0644"
   when:
     - kolla_copy_ca_into_containers | bool
+    - not (item.value.iterate | default(False)) | bool
   with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
 
+# NOTE(dougszu): For current iterable services, we only need CA certs
+- name: "Check containers that require iteration for {{ kolla_role_name | default(project_name) }}"
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ project_services | select_services_enabled_and_mapped_to_host | dict2items }}"
+  loop_control:
+    loop_var: outer_item
+  when:
+    - (service.iterate | default(False)) | bool
+    - kolla_copy_ca_into_containers | bool
+
 - name: "Copying over backend internal TLS certificate for {{ project_name }}"
   vars:
     certs:
diff --git a/ansible/roles/service-check-containers/tasks/iterated.yml b/ansible/roles/service-check-containers/tasks/iterated.yml
index bd266b115..c4f28e6d4 100644
--- a/ansible/roles/service-check-containers/tasks/iterated.yml
+++ b/ansible/roles/service-check-containers/tasks/iterated.yml
@@ -23,6 +23,12 @@
     command: "{{ service.command | default(omit) }}"
     cgroupns_mode: "{{ service.cgroupns_mode | default(omit) }}"
   loop: "{{ range(1, (iterate_count | int) + 1) | list }}"
+  when:
+    # NOTE(dougszu): Due to issues with nova-compute-ironic HA, we can't have
+    # more than one instance for a given 'item'. Eg. nova_compute_ironic_1
+    # should never run on more than one host.
+    # See: https://specs.openstack.org/openstack/nova-specs/specs/2024.1/implemented/ironic-shards.html
+    - service_name != 'nova-compute-ironic' or inventory_hostname in groups[nova_cell_compute_ironic_group ~ '-' ~ item]
   register: container_check
 
 - name: "Store changed containers to pass to the handler"
diff --git a/ansible/roles/service-precheck/tasks/iterated.yml b/ansible/roles/service-precheck/tasks/iterated.yml
new file mode 100644
index 000000000..d9157ddce
--- /dev/null
+++ b/ansible/roles/service-precheck/tasks/iterated.yml
@@ -0,0 +1,13 @@
+---
+- name: "Validate inventory groups (iterated) for {{ project_name }}"
+  vars:
+    service_name: "{{ outer_item.key }}"
+    service: "{{ outer_item.value }}"
+    service_group: "{{ service.group ~ '-' ~ item }}"
+  ansible.builtin.fail:
+    msg: >-
+      Ansible inventory does not contain the expected group {{ service_group }}
+      for service {{ service_name }} in {{ project_name }}.
+  when:
+    - service_group not in groups
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
diff --git a/ansible/roles/service-precheck/tasks/main.yml b/ansible/roles/service-precheck/tasks/main.yml
index 58448555e..81b90f6a5 100644
--- a/ansible/roles/service-precheck/tasks/main.yml
+++ b/ansible/roles/service-precheck/tasks/main.yml
@@ -9,7 +9,19 @@
       for service {{ service_name }} in {{ project_name }}.
   loop: "{{ query('dict', service_precheck_services) }}"
   when:
+    - not (service.iterate | default(False)) | bool
     - "'group' in service"
     - service.group not in groups
   loop_control:
     label: "{{ service_name }}"
+
+- name: Include tasks
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ query('dict', service_precheck_services) }}"
+  loop_control:
+    loop_var: outer_item
+  when:
+    - (service.iterate | default(False)) | bool
+    - "'group' in service"
diff --git a/ansible/roles/service-stop/tasks/iterated.yml b/ansible/roles/service-stop/tasks/iterated.yml
index f25bf0c26..cf4e8e489 100644
--- a/ansible/roles/service-stop/tasks/iterated.yml
+++ b/ansible/roles/service-stop/tasks/iterated.yml
@@ -11,4 +11,5 @@
     ignore_missing: "{{ kolla_action_stop_ignore_missing | bool }}"
   when:
     - service.container_name not in skip_stop_containers
+    - service_name != 'nova-compute-ironic' or inventory_hostname in groups[service.group ~ '-' ~ item]
   loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
diff --git a/doc/source/admin/advanced-configuration.rst b/doc/source/admin/advanced-configuration.rst
index 7ece68e3a..c85a498dc 100644
--- a/doc/source/admin/advanced-configuration.rst
+++ b/doc/source/admin/advanced-configuration.rst
@@ -178,6 +178,113 @@ operator needs to create ``/etc/kolla/config/global.conf`` with content:
    [database]
    max_pool_size = 100
 
+Large baremetal deployments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Out of the box, a typical Kolla Ansible deployment can support managing a few
+hundred baremetal nodes. Beyond this number, it becomes necessary to configure
+scaling mechanisms built into Nova and Ironic.
+
+There are two mechanisms to consider:
+  - shards
+  - conductor groups
+
+Please see the Ironic documentation for further `information
+<https://docs.openstack.org/ironic/latest/admin/availability-zones.html#common-deployment-patterns>`_.
+
+As an example, consider a deployment of 700 baremetal nodes spread over two
+distinct locations. Location 1 consists of 500 baremetal nodes, and location 2
+consists of 200 baremetal nodes.
+
+For location 1, all 500 nodes are placed into the same Ironic conductor
+group. Ironic conductors configured to use this group are deployed on each
+of the three nodes in the control plane. A single Nova Compute Ironic service
+would struggle to manage this many nodes, so two shards are created. Due to
+HA limitations with Nova Compute Ironic, a single instance is created for
+each shard, and the instances may be deployed to any node in the control
+plane.
+
+For location 2, no shards are required due to the smaller number of baremetal
+nodes. A single conductor group is configured. The result is that three Ironic
+conductors are deployed (one on each node in the control plane), and a single
+instance of Nova Compute Ironic configured to use this conductor group.
+Furthermore, for this location ``custom_host`` is used to override the
+automatically generated host field in the configuration. This is useful for
+migrating to the multi-instance configuration described here.
+
+Finally, Ironic and Nova services are deployed with no configured shards or
+conductor groups as a catch all.
+
+.. code-block:: yaml
+
+  nova_multi_compute_ironic_config:
+    - "shard_key": "shard_1"
+      "conductor_group": "location_1"
+    - "shard_key": "shard_2"
+      "conductor_group": "location_1"
+    - "conductor_group": "location_2"
+      "custom_host": "some_custom_host_field"
+    - {}
+
+.. note::
+
+   ``nova_multi_compute_ironic_config`` must be defined as a global variable
+   and not customised via hostvars.
+
+.. note::
+
+   If you currently have the ``nova-compute-ironic`` service deployed,
+   you must manually remove it from the controllers before migrating to the
+   new mode. You will need to ensure that you configure a compatible service
+   in ``nova_multi_compute_ironic_config`` to replace the old one. This will
+   likely involve setting the ``custom_host`` field at the very minimum.
+
+The placement of the above services is managed via the inventory. This
+allows flexible placement, should a controller fail.
+
+.. code-block:: yaml
+
+  [nova-compute-ironic-1]
+  controller-01
+  [nova-compute-ironic-2]
+  controller-02
+  [nova-compute-ironic-3]
+  controller-03
+  [nova-compute-ironic-4]
+  controller-03
+
+  [nova-compute-ironic:children]
+  nova-compute-ironic-1
+  nova-compute-ironic-2
+  nova-compute-ironic-3
+  nova-compute-ironic-4
+
+.. note::
+
+   The number of ``nova-compute-ironic-N`` groups must equal the length of
+   ``nova_multi_compute_ironic_config``. This variable is an ordered list,
+   where the first item maps to ``nova-compute-ironic-1``, the second item
+   to ``nova-compute-ironic-2`` and so forth.
+
+.. note::
+
+   A 'cleanup' function is provided which assists in 'failing over'
+   service instances to other hosts. For example, if ``controller-01``
+   was to be taken down for maintenance, replacing it with ``controller-02``
+   in the ``nova-compute-ironic-1`` group and re-deploying the ``nova``
+   service would result in the service being removed from ``controller-01``
+   and re-deployed on ``controller-02``. The cleanup function cannot
+   currently handle downsizing the number of service instances. In this case
+   the redundant service instances must be cleaned up manually. For obvious
+   reasons the cleanup function cannot remove a service from a failed node,
+   and in such a case, you must be careful not to allow a second identical
+   service instance to start when the failed node is recovered.
+
+The ``nova-compute-ironic`` service instances may be configured
+individually via the existing override mechanism. For example, the
+creation of ``nova/nova-compute-ironic-1.conf`` will allow variables
+for that specific service to be overridden.
+
 OpenStack policy customisation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/kolla_ansible/nova_filters.py b/kolla_ansible/nova_filters.py
index 4bb5cbdf6..f9be3eb98 100644
--- a/kolla_ansible/nova_filters.py
+++ b/kolla_ansible/nova_filters.py
@@ -15,6 +15,8 @@
 import jinja2
 import re
 
+from kolla_ansible import exception
+
 
 def extract_cell(list_cells_cli_output, cell_name):
     """Extract cell settings from nova_manage CLI
@@ -85,8 +87,57 @@ def namespace_haproxy_for_cell(services, cell_name):
     return services
 
 
+def get_expected_ironic_compute_services(ironic_compute_batch_info,
+                                         multi_compute_conf,
+                                         nova_cell_compute_ironic_group):
+    """Return a List of expected Ironic compute services
+
+      :param ironic_compute_batch_info: A dict containing configuration
+             information about Ironic Compute hosts.
+      :param multi_compute_conf: Ironic multi-compute config
+      :param nova_cell_compute_ironic_group: Ansible group name for
+             nova-compute-ironic. Eg. 'nova-compute-ironic'.
+      :returns: A list of Nova Compute Ironic service hostnames.
+    """
+    classic_info = ironic_compute_batch_info['classic']
+    multi_compute_info = ironic_compute_batch_info['multi']
+    if len(multi_compute_conf) == 0:
+        return [f'{host}-ironic' for host in classic_info]
+
+    expected_services = []
+    group_prefix = nova_cell_compute_ironic_group + '-'
+    for host in multi_compute_info:
+        iterate_vars = [
+            int(group.split(group_prefix)[1])
+            for group in host if group_prefix in group]
+        for i in iterate_vars:
+            try:
+                service_conf = multi_compute_conf[i - 1]
+            except IndexError:
+                raise exception.FilterError(
+                    "Unable to look up multi-compute Ironic config for "
+                    "{g}-{i} inventory group. Check that "
+                    "nova_multi_compute_ironic_config has an entry for "
+                    "each defined group.".format(
+                        g=nova_cell_compute_ironic_group,
+                        i=i))
+            expected_services.append(
+                _get_expected_compute_service_name(service_conf))
+    return expected_services
+
+
+def _get_expected_compute_service_name(service_conf):
+    custom_host = service_conf.get('custom_host')
+    if custom_host:
+        return f"{custom_host}-ironic"
+    cg = service_conf.get('conductor_group', 'default')
+    sk = service_conf.get('shard_key', 'default')
+    return f"{cg}-{sk}-ironic"
+
+
 def get_filters():
     return {
         "extract_cell": extract_cell,
         "namespace_haproxy_for_cell": namespace_haproxy_for_cell,
+        "get_expected_ironic_compute_services": get_expected_ironic_compute_services,  # noqa
     }
diff --git a/kolla_ansible/tests/unit/test_nova_filters.py b/kolla_ansible/tests/unit/test_nova_filters.py
index e8d5c507a..d436b0ef1 100644
--- a/kolla_ansible/tests/unit/test_nova_filters.py
+++ b/kolla_ansible/tests/unit/test_nova_filters.py
@@ -73,6 +73,92 @@ class TestFilters(unittest.TestCase):
         self.assertRaisesRegex(jinja2.TemplateRuntimeError, 'duplicates',
                                self._test_extract_cell, test_data, 'cell0001')
 
+    def test_get_expected_ironic_compute_services_multi_compute(self):
+        example_ironic_compute_conf = {
+            # We expect the classic config to be ignored
+            'classic': ['custom-host-nova-compute'],
+            # We have a batch of 2 Ironic compute hosts which should
+            # run a total of 3 Nova compute Ironic instances
+            'multi': [
+                ['nova-compute-ironic-1',
+                 'nova-compute-ironic-2',
+                 'nova-compute-ironic',
+                 'something'],
+                ['nova-compute-ironic-3',
+                    'nova-compute-ironic',
+                    'something'],
+            ]
+        }
+        multi_conf = [
+            {"custom_host": "some_custom_host"},
+            {"shard_key": "shard_1", "conductor_group": "location_1"},
+            {"shard_key": "shard_2", "conductor_group": "location_1"},
+            {"conductor_group": "location_2"},
+            {"shard_key": "shard_1"}
+        ]
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, multi_conf, 'nova-compute-ironic')
+        expected = [
+            'some_custom_host-ironic',
+            'location_1-shard_1-ironic',
+            'location_1-shard_2-ironic',
+        ]
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_no_compute(self):
+        example_ironic_compute_conf = {
+            'classic': [],
+            'multi': []
+        }
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, [], 'nova-compute-ironic')
+        expected = []
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_classic_compute(self):
+        example_ironic_compute_conf = {
+            'classic': ['custom-foo'],
+            'multi': []
+        }
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, [], 'nova-compute-ironic')
+        expected = ['custom-foo-ironic']
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_classic_compute_multi(self):
+        # NOTE(dougszu): This isn't a recommended configuration
+        example_ironic_compute_conf = {
+            'classic': ['custom-foo', 'ctrl02', 'ctrl03'],
+            'multi': []
+        }
+        actual = filters.get_expected_ironic_compute_services(
+            example_ironic_compute_conf, [], 'nova-compute-ironic')
+        expected = [
+            'custom-foo-ironic',
+            'ctrl02-ironic',
+            'ctrl03-ironic',
+        ]
+        self.assertListEqual(actual, expected)
+
+    def test_get_expected_ironic_compute_services_bad_config(self):
+        example_ironic_compute_conf = {
+            'classic': [],
+            'multi': [
+                ['my-custom-group-1',
+                 'my-custom-group-2',
+                 'my-custom-group',
+                 'something'],
+            ]
+        }
+        multi_conf = [
+            {"shard_key": "shard_1", "conductor_group": "location_1"},
+        ]
+        with self.assertRaisesRegex(Exception,
+                                    'Unable to look up multi-compute Ironic '
+                                    'config for my-custom-group-2'):
+            filters.get_expected_ironic_compute_services(
+                example_ironic_compute_conf, multi_conf, 'my-custom-group')
+
     def test_namespace_haproxy_for_cell_with_empty_name(self):
         example_services = {
             'nova-novncproxy': {
diff --git a/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml
new file mode 100644
index 000000000..c99523feb
--- /dev/null
+++ b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds support for deploying multiple instances of the Nova Compute Ironic
+    service on the same host. This is useful in large baremetal deployments.
+upgrade:
+  - |
+    If you are migrating to the new feature to deploy multiple instances of
+    Nova Compute Ironic, you are first required to manually remove the
+    existing Nova Compute Ironic Docker container, and config directories.
diff --git a/roles/openstack-clients/defaults/main.yml b/roles/openstack-clients/defaults/main.yml
index e1f53e279..034618843 100644
--- a/roles/openstack-clients/defaults/main.yml
+++ b/roles/openstack-clients/defaults/main.yml
@@ -11,7 +11,7 @@ openstack_clients_pip_packages:
   - package: python-heatclient
     enabled: true
   - package: python-ironicclient
-    enabled: "{{ scenario == 'ironic' }}"
+    enabled: "{{ scenario in ['ironic', 'multi-compute-ironic'] }}"
   - package: python-magnumclient
     enabled: "{{ scenario == 'magnum' }}"
   - package: python-neutronclient
diff --git a/tests/run.yml b/tests/run.yml
index 8dd6c62c6..efa817e24 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -208,10 +208,10 @@
         # ironic.conf
         - src: "tests/templates/ironic-overrides.j2"
           dest: /etc/kolla/config/ironic.conf
-          when: "{{ scenario == 'ironic' }}"
+          when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}"
         - src: "tests/templates/tenks-deploy-config.yml.j2"
           dest: "{{ ansible_env.HOME }}/tenks.yml"
-          when: "{{ scenario == 'ironic' }}"
+          when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}"
         # keystone-federation config
         - src: "tests/templates/keystone-federation/idp.example.org%2Frealms%2Fexample.conf.j2"
           dest: "/etc/kolla/config/keystone-httpd/federation/oidc/metadata/idp.example.org%2Frealms%2Fexample.conf"
@@ -243,7 +243,7 @@
               dest: ironic-agent.initramfs
             - src: "ipa-centos9-{{ zuul.branch | replace('/', '-') }}.kernel"
               dest: ironic-agent.kernel
-      when: scenario == "ironic"
+      when: scenario == "ironic" or scenario == "multi-compute-ironic"
 
     - block:
         - name: Slurp requirements.yml
@@ -427,7 +427,7 @@
             EXT_NET_GATEWAY: "{{ neutron_external_network_prefix }}1"
             EXT_NET_DEMO_ROUTER_ADDR: "{{ neutron_external_network_prefix }}10"
             SCENARIO: "{{ scenario }}"
-          when: openstack_core_tested or scenario in ['ironic', 'magnum', 'nfv', 'zun', 'octavia']
+          when: openstack_core_tested or scenario in ['ironic', 'multi-compute-ironic' ,'magnum', 'nfv', 'zun', 'octavia']
 
         - name: Test OVN VPNaaS
           import_role:
@@ -494,7 +494,8 @@
                 chdir: "{{ kolla_ansible_src_dir }}"
               environment:
                 TLS_ENABLED: "{{ tls_enabled }}"
-          when: scenario == "ironic"
+                MULTI_COMPUTE_IRONIC: "{{ scenario == 'multi-compute-ironic' }}"
+          when: scenario == "ironic" or scenario == "multi-compute-ironic"
 
         - name: Run test-magnum.sh script
           script:
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index f5f005acd..ca2ad562e 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -138,6 +138,22 @@ enable_proxysql: "yes"
 enable_prometheus_proxysql_exporter: "yes"
 {% endif %}
 
+{% if scenario == "multi-compute-ironic" %}
+nova_multi_compute_ironic_config:
+  - {}
+  - "shard_key": "shard_1"
+    "conductor_group": "location_1"
+  - "shard_key": "shard_2"
+    "conductor_group": "location_1"
+  - "conductor_group": "location_2"
+
+enable_ironic: "yes"
+enable_ironic_pxe_filter: "yes"
+ironic_dnsmasq_dhcp_ranges:
+  - range: "10.42.0.2,10.42.0.254,255.255.255.0"
+ironic_internal_endpoint: http://192.0.2.1:6385
+{% endif %}
+
 {% if scenario == "mariadb" %}
 enable_fluentd: "yes"
 enable_mariadb: "yes"
diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2
index 2991518ce..a3bea7213 100644
--- a/tests/templates/inventory.j2
+++ b/tests/templates/inventory.j2
@@ -6,7 +6,7 @@
 # primary and secondary while the parent group (control) uses only primary.
 
 [control]
-{% if scenario == 'masakari' %}
+{% if scenario in ['masakari', 'multi-compute-ironic'] %}
 {%     for host in hostvars if host in ['primary'] %}
 {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }}
 {%     endfor %}
@@ -299,8 +299,25 @@ nova
 [nova-spicehtml5proxy:children]
 nova
 
+{% if scenario == 'multi-compute-ironic' %}
+[nova-compute-ironic-1]
+primary
+[nova-compute-ironic-2]
+secondary1
+[nova-compute-ironic-3]
+secondary2
+[nova-compute-ironic-4]
+secondary2
+
+[nova-compute-ironic:children]
+nova-compute-ironic-1
+nova-compute-ironic-2
+nova-compute-ironic-3
+nova-compute-ironic-4
+{% else %}
 [nova-compute-ironic:children]
 nova
+{% endif %}
 
 [nova-serialproxy:children]
 nova
diff --git a/tests/templates/ironic-overrides.j2 b/tests/templates/ironic-overrides.j2
index be944b307..9603d6b2c 100644
--- a/tests/templates/ironic-overrides.j2
+++ b/tests/templates/ironic-overrides.j2
@@ -2,6 +2,16 @@
 enabled_inspect_interfaces = no-inspect, agent
 default_inspect_interface = agent
 
+{% if scenario == "multi-compute-ironic" %}
+{%     raw %}
+{%     set cg = 'location_1' if inventory_hostname == 'secondary1' else 'location_2' if inventory_hostname == 'secondary2' %}
+{%     if cg %}
+[conductor]
+conductor_group = {{ cg }}
+{%     endif %}
+{%     endraw %}
+{% endif %}
+
 [neutron]
 cleaning_network = public1
 inspection_network = public1
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 0758a8067..7cb2ec76f 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -4,6 +4,7 @@
     templates:
       - ansible-role-jobs
       - check-requirements
+      - kolla-ansible-scenario-multi-compute-ironic
       - kolla-ansible-scenario-aio
       - kolla-ansible-scenario-bifrost
       - kolla-ansible-scenario-cells
diff --git a/zuul.d/scenarios/multi-compute-ironic.yaml b/zuul.d/scenarios/multi-compute-ironic.yaml
new file mode 100644
index 000000000..814116a1f
--- /dev/null
+++ b/zuul.d/scenarios/multi-compute-ironic.yaml
@@ -0,0 +1,47 @@
+---
+- job:
+    name: kolla-ansible-multi-compute-ironic-base
+    parent: kolla-ansible-base
+    voting: false
+    files: !inherit
+      - ^ansible/group_vars/all/(nova|ironic).yml
+      - ^ansible/roles/(nova|nova-cell|ironic)/
+      - ^tests/deploy-tenks\.sh$
+      - ^tests/templates/ironic-overrides\.j2$
+      - ^tests/templates/tenks-deploy-config\.yml\.j2$
+      - ^tests/test-dashboard\.sh$
+      - ^tests/test-ironic\.sh$
+    required-projects:
+      - openstack/tenks
+    vars:
+      scenario: multi-compute-ironic
+      scenario_images_extra:
+        - ^dnsmasq
+        - ^ironic
+        - ^iscsid
+      tls_enabled: false
+
+- job:
+    name: kolla-ansible-debian-trixie-multi-compute-ironic
+    parent: kolla-ansible-multi-compute-ironic-base
+    nodeset: kolla-ansible-debian-trixie-multi-16GB
+
+- job:
+    name: kolla-ansible-rocky-10-multi-compute-ironic
+    parent: kolla-ansible-multi-compute-ironic-base
+    nodeset: kolla-ansible-rocky-10-multi-16GB
+
+- job:
+    name: kolla-ansible-ubuntu-noble-multi-compute-ironic
+    parent: kolla-ansible-multi-compute-ironic-base
+    nodeset: kolla-ansible-ubuntu-noble-multi-16GB
+
+- project-template:
+    name: kolla-ansible-scenario-multi-compute-ironic
+    description: |
+      Runs Kolla-Ansible Nova Multi Compute Ironic scenario jobs.
+    check:
+      jobs:
+        - kolla-ansible-debian-trixie-multi-compute-ironic
+        - kolla-ansible-rocky-10-multi-compute-ironic
+        - kolla-ansible-ubuntu-noble-multi-compute-ironic