diff options
27 files changed, 627 insertions, 19 deletions
diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml index 6979408e9..790190fb0 100644 --- a/ansible/roles/nova-cell/defaults/main.yml +++ b/ansible/roles/nova-cell/defaults/main.yml @@ -72,6 +72,8 @@ nova_cell_services: volumes: "{{ nova_compute_ironic_default_volumes + nova_compute_ironic_extra_volumes }}" dimensions: "{{ nova_compute_ironic_dimensions }}" healthcheck: "{{ nova_compute_ironic_healthcheck }}" + iterate: "{{ nova_multi_compute_ironic_config | length > 0 }}" + iterate_var: "{{ nova_multi_compute_ironic_config | length }}" #################### # Config Validate @@ -501,7 +503,7 @@ nova_compute_default_volumes: - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}" - "{{ kolla_dev_repos_directory ~ '/nova:/dev-mode/nova' if nova_dev_mode | bool else '' }}" nova_compute_ironic_default_volumes: - - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro" + - "{{ node_config_directory }}/nova-compute-ironic{{ '-' ~ item if item|default(0)|int > 0 else '' }}/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" - "{{ '/dev/shm:/dev/shm' }}" @@ -675,6 +677,17 @@ nova_libvirt_cleanup_remove_volumes: false # Database nova_cell_database_enable_tls_internal: "{{ database_enable_tls_internal | bool }}" +############################## +# Nova Compute Ironic scaling +############################## + +# The following option can be used to deploy multiple instances of Nova +# Compute Ironic per host. This is useful for deployments with a large +# number of baremetal nodes (>200). Please see the advanced documentation +# section for further details. + +nova_multi_compute_ironic_config: [] + ################### # Copy certificates ################### diff --git a/ansible/roles/nova-cell/handlers/main.yml b/ansible/roles/nova-cell/handlers/main.yml index a67a1231f..c6ac28da9 100644 --- a/ansible/roles/nova-cell/handlers/main.yml +++ b/ansible/roles/nova-cell/handlers/main.yml @@ -162,16 +162,21 @@ vars: service_name: "nova-compute-ironic" service: "{{ nova_cell_services[service_name] }}" + item: "{{ changed_container.item }}" + container_name_postfix: "{{ '' if changed_container == 'legacy' else '_' ~ item }}" become: true kolla_container: action: "recreate_or_restart_container" common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" + name: "{{ service.container_name + container_name_postfix }}" image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes | reject('equalto', '') | list }}" dimensions: "{{ service.dimensions }}" healthcheck: "{{ service.healthcheck | default(omit) }}" + loop_control: + loop_var: changed_container + loop: "{{ nova_compute_ironic_changed_containers | default(['legacy']) }}" # nova-compute-fake is special. It will start multi numbers of container # so put all variables here rather than defaults/main.yml file diff --git a/ansible/roles/nova-cell/tasks/cleanup.yml b/ansible/roles/nova-cell/tasks/cleanup.yml new file mode 100644 index 000000000..0f0952823 --- /dev/null +++ b/ansible/roles/nova-cell/tasks/cleanup.yml @@ -0,0 +1,26 @@ +--- +- name: Stop and remove containers for unmanaged services + become: true + vars: + service_name: "nova-compute-ironic" + service: "{{ nova_cell_services[service_name] }}" + kolla_container: + action: "stop_and_remove_container" + name: "{{ service.container_name ~ '_' ~ item }}" + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - service.iterate | bool + - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item] + +- name: Removing config for unmanaged services + vars: + service_name: "nova-compute-ironic" + service: "{{ nova_cell_services[service_name] }}" + ansible.builtin.file: + path: "{{ node_config_directory }}/{{ service_name ~ '-' ~ item }}" + state: "absent" + become: true + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - service.iterate | bool + - inventory_hostname not in groups[nova_cell_compute_ironic_group ~ '-' ~ item] diff --git a/ansible/roles/nova-cell/tasks/config-compute-ironic.yml b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml new file mode 100644 index 000000000..0f1c05a9f --- /dev/null +++ b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml @@ -0,0 +1,67 @@ +--- +- name: "Ensuring config directories exist for Nova Compute Ironic container {{ item }}" + vars: + nova_compute_ironic_id: "{{ item }}" + ansible.builtin.file: + path: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}" + state: "directory" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0770" + become: true + +- name: "Copying over config.json files for Nova Compute Ironic container {{ item }}" + vars: + nova_compute_ironic_id: "{{ item }}" + ansible.builtin.template: + src: "nova-compute-ironic.json.j2" + dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/config.json" + mode: "0660" + become: true + +- name: "Get Nova Compute Ironic multi-instance configuration for instance {{ item }}" + ansible.builtin.set_fact: + nova_multi_compute_ironic_instance_config: "{{ nova_multi_compute_ironic_config[(item | int) - 1] }}" + +- name: "Generate config files for Nova Compute Ironic {{ item }}" + vars: + service_name: "nova-compute-ironic" + nova_compute_shard_key: "{{ nova_multi_compute_ironic_instance_config['shard_key'] | default('default') }}" + nova_compute_conductor_group: "{{ nova_multi_compute_ironic_instance_config['conductor_group'] | default('default') }}" + nova_compute_ironic_custom_host: "{{ nova_multi_compute_ironic_instance_config['custom_host'] | default('') }}" + nova_compute_ironic_id: "{{ item }}" + merge_configs: + sources: + - "{{ role_path }}/templates/nova.conf.j2" + - "{{ node_custom_config }}/global.conf" + - "{{ node_custom_config }}/nova.conf" + - "{{ node_custom_config }}/nova/{{ service_name }}.conf" + - "{{ node_custom_config }}/nova/{{ service_name }}-{{ item }}.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/nova.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}-{{ item }}.conf" + dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/nova.conf" + mode: "0660" + become: true + +- name: "Copying over existing policy file for item {{ item }}" + vars: + service_name: "nova-compute-ironic" + ansible.builtin.template: + src: "{{ nova_policy_file_path }}" + dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/{{ nova_policy_file }}" + mode: "0660" + become: true + when: + - nova_policy_file is defined + +- name: "Copying over vendordata file for Nova Compute Ironic {{ item }}" + vars: + service_name: "nova-compute-ironic" + ansible.builtin.copy: + src: "{{ vendordata_file_path }}" + dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/vendordata.json" + mode: "0660" + become: true + when: + - vendordata_file_path is defined diff --git a/ansible/roles/nova-cell/tasks/config.yml b/ansible/roles/nova-cell/tasks/config.yml index 87104f89e..7f0cfca22 100644 --- a/ansible/roles/nova-cell/tasks/config.yml +++ b/ansible/roles/nova-cell/tasks/config.yml @@ -7,6 +7,8 @@ owner: "{{ config_owner_user }}" group: "{{ config_owner_group }}" mode: "0770" + when: + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" - name: Copying over TLS certificates @@ -58,6 +60,8 @@ src: "{{ item.key }}.json.j2" dest: "{{ node_config_directory }}/{{ item.key }}/config.json" mode: "0660" + when: + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" - name: Copying over nova.conf @@ -76,8 +80,18 @@ mode: "0660" when: - item.key in nova_cell_services_require_nova_conf + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" +- name: Copying over config for Nova compute Ironic multi-instance + vars: + service: "{{ nova_cell_services['nova-compute-ironic'] }}" + ansible.builtin.include_tasks: config-compute-ironic.yml + loop: "{{ range(1, nova_multi_compute_ironic_config | length + 1) | list }}" + when: + - nova_multi_compute_ironic_config | length > 0 + - inventory_hostname in groups[service.group ~ '-' ~ item] + - name: Copying over Nova compute provider config become: true vars: @@ -196,6 +210,7 @@ when: - nova_policy_file is defined - item.key in nova_cell_services_require_policy_json + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" - name: Copying over vendordata file to containers @@ -209,6 +224,7 @@ when: - vendordata_file_path is defined - service | service_enabled_and_mapped_to_host + - item != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_items: - nova-compute - nova-compute-ironic diff --git a/ansible/roles/nova-cell/tasks/deploy.yml b/ansible/roles/nova-cell/tasks/deploy.yml index 2a416d8d4..288531557 100644 --- a/ansible/roles/nova-cell/tasks/deploy.yml +++ b/ansible/roles/nova-cell/tasks/deploy.yml @@ -6,6 +6,10 @@ - name: Import tasks from version-check.yml ansible.builtin.import_tasks: version-check.yml +# NOTE(dougszu): Currently only covers nova-compute-ironic +- name: Cleanup stale configuration + ansible.builtin.import_tasks: cleanup.yml + - name: Import tasks from config-host.yml ansible.builtin.import_tasks: config-host.yml diff --git a/ansible/roles/nova-cell/tasks/precheck.yml b/ansible/roles/nova-cell/tasks/precheck.yml index a509e790c..9364c619e 100644 --- a/ansible/roles/nova-cell/tasks/precheck.yml +++ b/ansible/roles/nova-cell/tasks/precheck.yml @@ -30,6 +30,20 @@ - groups[nova_cell_compute_group] | length < 1 - not nova_compute_ironic.enabled | bool +- name: Checking nova multi-compute-ironic inventory + vars: + nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}" + ansible.builtin.assert: + msg: > + Each nova-compute-ironic-N group must contain a single host. + that: + - groups[nova_cell_compute_ironic_group ~ '-' ~ item] | length == 1 + loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}" + run_once: true + when: + - nova_compute_ironic.iterate | bool + - nova_compute_ironic.enabled | bool + - name: Checking free port for Nova NoVNC Proxy vars: service: "{{ nova_cell_services['nova-novncproxy'] }}" diff --git a/ansible/roles/nova-cell/tasks/reload.yml b/ansible/roles/nova-cell/tasks/reload.yml index 5724effdf..5f5c05b4e 100644 --- a/ansible/roles/nova-cell/tasks/reload.yml +++ b/ansible/roles/nova-cell/tasks/reload.yml @@ -37,4 +37,28 @@ when: - kolla_action == 'upgrade' - service | service_enabled_and_mapped_to_host + - item != 'nova-compute-ironic' or not service.iterate | bool with_items: "{{ nova_cell_services_require_nova_conf }}" + +- name: Reload nova-compute-ironic to remove RPC version cap (iterated) + vars: + nova_compute_ironic: "{{ nova_cell_services['nova-compute-ironic'] }}" + become: true + kolla_container: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ nova_compute_ironic.container_name ~ '_' ~ item }}" + image: "{{ nova_compute_ironic.image }}" + environment: "{{ nova_compute_ironic.environment | default(omit) }}" + pid_mode: "{{ nova_compute_ironic.pid_mode | default('') }}" + ipc_mode: "{{ nova_compute_ironic.ipc_mode | default(omit) }}" + privileged: "{{ nova_compute_ironic.privileged | default(False) }}" + volumes: "{{ nova_compute_ironic.volumes | reject('equalto', '') | list }}" + dimensions: "{{ nova_compute_ironic.dimensions }}" + healthcheck: "{{ nova_compute_ironic.healthcheck | default(omit) }}" + when: + - kolla_action == 'upgrade' + - nova_compute_ironic | service_enabled + - nova_compute_ironic.iterate | bool + - inventory_hostname in groups[nova_compute_ironic.group ~ '-' ~ item] + loop: "{{ range(1, (nova_compute_ironic.iterate_var | int) + 1) | list }}" diff --git a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml index d930f8dea..eb7e80e83 100644 --- a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml +++ b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml @@ -13,13 +13,14 @@ {{ virt_computes_in_batch | map('extract', hostvars, ['ansible_facts', 'nodename']) | list }} - # For ironic, use {{ansible_facts.hostname}}-ironic since this is what we - # configure for [DEFAULT] host in nova.conf. ironic_compute_service_hosts: >- {{ ironic_computes_in_batch | - map('extract', hostvars) | json_query('[].nova_compute_ironic_custom_host || [].ansible_facts.hostname') | - map('regex_replace', '^(.*)$', '\1-ironic') | - list }} + map('extract', hostvars) | + json_query( + '{classic: [].nova_compute_ironic_custom_host || [].ansible_facts.hostname, + multi: [].group_names}') | + get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }} + expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}" block: - name: Waiting for nova-compute services to register themselves @@ -73,13 +74,19 @@ failed_compute_service_hosts: >- {{ expected_compute_service_hosts | difference(nova_compute_service_hosts) | list }} # Whether any compute services failed on this host. + ironic_compute_service_config_for_host: + classic: "{{ [nova_compute_ironic_custom_host | default(ansible_facts.hostname)] if inventory_hostname in ironic_computes_in_batch else [] }}" + multi: "{{ [group_names] }}" + expected_ironic_compute_services: >- + {{ ironic_compute_service_config_for_host | get_expected_ironic_compute_services(nova_multi_compute_ironic_config, nova_cell_compute_ironic_group) }} any_failed_services: >- {{ (ansible_facts.nodename | default()) in failed_compute_service_hosts or - ((ansible_facts.hostname | default()) ~ "-ironic") in failed_compute_service_hosts }} + expected_ironic_compute_services | intersect(failed_compute_service_hosts) | length > 0 }} + ansible.builtin.fail: msg: >- - The Nova compute service failed to register itself on the following - hosts: {{ failed_compute_service_hosts | join(',') }} + The following Nova compute services failed to register + themselves: {{ failed_compute_service_hosts | join(',') }} when: >- any_failed_services or (nova_compute_registration_fatal | bool and diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2 index 2950dfcc4..7d30dcfd0 100644 --- a/ansible/roles/nova-cell/templates/nova.conf.j2 +++ b/ansible/roles/nova-cell/templates/nova.conf.j2 @@ -8,9 +8,27 @@ state_path = /var/lib/nova allow_resize_to_same_host = true +{% set nova_compute_shard_key = nova_compute_shard_key | default("") %} +{% set nova_compute_conductor_group = nova_compute_conductor_group | default("") %} +{% set nova_compute_ironic_custom_host = nova_compute_ironic_custom_host | default("") %} {% if service_name == "nova-compute-ironic" %} -host={{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic +{% if nova_compute_ironic_custom_host | length > 0 %} +{# NOTE(dougszu): This is provided for backwards compatibility #} +{% set host_value = nova_compute_ironic_custom_host ~ "-ironic" %} +{% elif nova_compute_conductor_group | length > 0 or nova_compute_shard_key | length > 0 %} +{% set host_value = (nova_compute_conductor_group ~ "-" ~ nova_compute_shard_key) ~ "-ironic" %} +{% else %} +{# NOTE(dougszu): This is to support legacy behaviour #} +{% set host_value = ansible_facts.hostname ~ "-ironic" %} +{% endif %} +host = {{ host_value }} + +{% if nova_compute_ironic_id is defined %} +log_file = /var/log/kolla/nova/nova-compute-ironic-{{ nova_compute_ironic_id }}.log +{% else %} log_file = /var/log/kolla/nova/nova-compute-ironic.log +{% endif %} + compute_driver = ironic.IronicDriver ram_allocation_ratio = 1.0 reserved_host_memory_mb = 0 @@ -101,6 +119,19 @@ project_name = service user_domain_name = {{ default_user_domain_name }} project_domain_name = {{ default_project_domain_name }} endpoint_override = {{ ironic_internal_endpoint }}/v1 + +{% if nova_compute_shard_key is not in ['default', ''] %} +shard = {{ nova_compute_shard_key }} +{% endif %} +{% if nova_compute_conductor_group is not in ['default', ''] %} +conductor_group = {{ nova_compute_conductor_group }} +{% endif %} +{% if nova_compute_conductor_group is not in ['default', ''] and nova_compute_shard_key is in ['default', ''] %} +{# NOTE(dougszu): In this case only, Nova Compute Ironic won't start unless the peer_list #} +{# is populated. peer_list is a deprecated config option and should never have more than #} +{# one host in it. This appears to be a bug. #} +peer_list = {{ host_value }} +{% endif %} {% endif %} [oslo_concurrency] @@ -180,7 +211,7 @@ driver = noop [oslo_messaging_rabbit] use_queue_manager = true {% if service_name == "nova-compute-ironic" %} -hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic +hostname = {{ host_value }} {% endif %} heartbeat_in_pthread = false {% if om_enable_rabbitmq_tls | bool %} diff --git a/ansible/roles/service-cert-copy/tasks/iterated.yml b/ansible/roles/service-cert-copy/tasks/iterated.yml new file mode 100644 index 000000000..4da5741fe --- /dev/null +++ b/ansible/roles/service-cert-copy/tasks/iterated.yml @@ -0,0 +1,10 @@ +--- +- name: "Copying over extra CA certificates for {{ project_name }}" + become: true + ansible.builtin.copy: + src: "{{ kolla_certificates_dir }}/ca/" + dest: "{{ node_config_directory }}/{{ outer_item.key }}-{{ item }}/ca-certificates" + mode: "0644" + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - inventory_hostname in groups[service.group ~ '-' ~ item] diff --git a/ansible/roles/service-cert-copy/tasks/main.yml b/ansible/roles/service-cert-copy/tasks/main.yml index d96bf0ca6..ef7ac6717 100644 --- a/ansible/roles/service-cert-copy/tasks/main.yml +++ b/ansible/roles/service-cert-copy/tasks/main.yml @@ -7,8 +7,21 @@ mode: "0644" when: - kolla_copy_ca_into_containers | bool + - not (item.value.iterate | default(False)) | bool with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}" +# NOTE(dougszu): For current iterable services, we only need CA certs +- name: "Check containers that require iteration for {{ kolla_role_name | default(project_name) }}" + vars: + service: "{{ outer_item.value }}" + ansible.builtin.include_tasks: iterated.yml + loop: "{{ project_services | select_services_enabled_and_mapped_to_host | dict2items }}" + loop_control: + loop_var: outer_item + when: + - (service.iterate | default(False)) | bool + - kolla_copy_ca_into_containers | bool + - name: "Copying over backend internal TLS certificate for {{ project_name }}" vars: certs: diff --git a/ansible/roles/service-check-containers/tasks/iterated.yml b/ansible/roles/service-check-containers/tasks/iterated.yml index bd266b115..c4f28e6d4 100644 --- a/ansible/roles/service-check-containers/tasks/iterated.yml +++ b/ansible/roles/service-check-containers/tasks/iterated.yml @@ -23,6 +23,12 @@ command: "{{ service.command | default(omit) }}" cgroupns_mode: "{{ service.cgroupns_mode | default(omit) }}" loop: "{{ range(1, (iterate_count | int) + 1) | list }}" + when: + # NOTE(dougszu): Due to issues with nova-compute-ironic HA, we can't have + # more than one instance for a given 'item'. Eg. nova_compute_ironic_1 + # should never run on more than one host. + # See: https://specs.openstack.org/openstack/nova-specs/specs/2024.1/implemented/ironic-shards.html + - service_name != 'nova-compute-ironic' or inventory_hostname in groups[nova_cell_compute_ironic_group ~ '-' ~ item] register: container_check - name: "Store changed containers to pass to the handler" diff --git a/ansible/roles/service-precheck/tasks/iterated.yml b/ansible/roles/service-precheck/tasks/iterated.yml new file mode 100644 index 000000000..d9157ddce --- /dev/null +++ b/ansible/roles/service-precheck/tasks/iterated.yml @@ -0,0 +1,13 @@ +--- +- name: "Validate inventory groups (iterated) for {{ project_name }}" + vars: + service_name: "{{ outer_item.key }}" + service: "{{ outer_item.value }}" + service_group: "{{ service.group ~ '-' ~ item }}" + ansible.builtin.fail: + msg: >- + Ansible inventory does not contain the expected group {{ service_group }} + for service {{ service_name }} in {{ project_name }}. + when: + - service_group not in groups + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" diff --git a/ansible/roles/service-precheck/tasks/main.yml b/ansible/roles/service-precheck/tasks/main.yml index 58448555e..81b90f6a5 100644 --- a/ansible/roles/service-precheck/tasks/main.yml +++ b/ansible/roles/service-precheck/tasks/main.yml @@ -9,7 +9,19 @@ for service {{ service_name }} in {{ project_name }}. loop: "{{ query('dict', service_precheck_services) }}" when: + - not (service.iterate | default(False)) | bool - "'group' in service" - service.group not in groups loop_control: label: "{{ service_name }}" + +- name: Include tasks + vars: + service: "{{ outer_item.value }}" + ansible.builtin.include_tasks: iterated.yml + loop: "{{ query('dict', service_precheck_services) }}" + loop_control: + loop_var: outer_item + when: + - (service.iterate | default(False)) | bool + - "'group' in service" diff --git a/ansible/roles/service-stop/tasks/iterated.yml b/ansible/roles/service-stop/tasks/iterated.yml index f25bf0c26..cf4e8e489 100644 --- a/ansible/roles/service-stop/tasks/iterated.yml +++ b/ansible/roles/service-stop/tasks/iterated.yml @@ -11,4 +11,5 @@ ignore_missing: "{{ kolla_action_stop_ignore_missing | bool }}" when: - service.container_name not in skip_stop_containers + - service_name != 'nova-compute-ironic' or inventory_hostname in groups[service.group ~ '-' ~ item] loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" diff --git a/doc/source/admin/advanced-configuration.rst b/doc/source/admin/advanced-configuration.rst index 7ece68e3a..c85a498dc 100644 --- a/doc/source/admin/advanced-configuration.rst +++ b/doc/source/admin/advanced-configuration.rst @@ -178,6 +178,113 @@ operator needs to create ``/etc/kolla/config/global.conf`` with content: [database] max_pool_size = 100 +Large baremetal deployments +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Out of the box, a typical Kolla Ansible deployment can support managing a few +hundred baremetal nodes. Beyond this number, it becomes necessary to configure +scaling mechanisms built into Nova and Ironic. + +There are two mechanisms to consider: + - shards + - conductor groups + +Please see the Ironic documentation for further `information +<https://docs.openstack.org/ironic/latest/admin/availability-zones.html#common-deployment-patterns>`_. + +As an example, consider a deployment of 700 baremetal nodes spread over two +distinct locations. Location 1 consists of 500 baremetal nodes, and location 2 +consists of 200 baremetal nodes. + +For location 1, all 500 nodes are placed into the same Ironic conductor +group. Ironic conductors configured to use this group are deployed on each +of the three nodes in the control plane. A single Nova Compute Ironic service +would struggle to manage this many nodes, so two shards are created. Due to +HA limitations with Nova Compute Ironic, a single instance is created for +each shard, and the instances may be deployed to any node in the control +plane. + +For location 2, no shards are required due to the smaller number of baremetal +nodes. A single conductor group is configured. The result is that three Ironic +conductors are deployed (one on each node in the control plane), and a single +instance of Nova Compute Ironic configured to use this conductor group. +Furthermore, for this location ``custom_host`` is used to override the +automatically generated host field in the configuration. This is useful for +migrating to the multi-instance configuration described here. + +Finally, Ironic and Nova services are deployed with no configured shards or +conductor groups as a catch all. + +.. code-block:: yaml + + nova_multi_compute_ironic_config: + - "shard_key": "shard_1" + "conductor_group": "location_1" + - "shard_key": "shard_2" + "conductor_group": "location_1" + - "conductor_group": "location_2" + "custom_host": "some_custom_host_field" + - {} + +.. note:: + + ``nova_multi_compute_ironic_config`` must be defined as a global variable + and not customised via hostvars. + +.. note:: + + If you currently have the ``nova-compute-ironic`` service deployed, + you must manually remove it from the controllers before migrating to the + new mode. You will need to ensure that you configure a compatible service + in ``nova_multi_compute_ironic_config`` to replace the old one. This will + likely involve setting the ``custom_host`` field at the very minimum. + +The placement of the above services is managed via the inventory. This +allows flexible placement, should a controller fail. + +.. code-block:: yaml + + [nova-compute-ironic-1] + controller-01 + [nova-compute-ironic-2] + controller-02 + [nova-compute-ironic-3] + controller-03 + [nova-compute-ironic-4] + controller-03 + + [nova-compute-ironic:children] + nova-compute-ironic-1 + nova-compute-ironic-2 + nova-compute-ironic-3 + nova-compute-ironic-4 + +.. note:: + + The number of ``nova-compute-ironic-N`` groups must equal the length of + ``nova_multi_compute_ironic_config``. This variable is an ordered list, + where the first item maps to ``nova-compute-ironic-1``, the second item + to ``nova-compute-ironic-2`` and so forth. + +.. note:: + + A 'cleanup' function is provided which assists in 'failing over' + service instances to other hosts. For example, if ``controller-01`` + was to be taken down for maintenance, replacing it with ``controller-02`` + in the ``nova-compute-ironic-1`` group and re-deploying the ``nova`` + service would result in the service being removed from ``controller-01`` + and re-deployed on ``controller-02``. The cleanup function cannot + currently handle downsizing the number of service instances. In this case + the redundant service instances must be cleaned up manually. For obvious + reasons the cleanup function cannot remove a service from a failed node, + and in such a case, you must be careful not to allow a second identical + service instance to start when the failed node is recovered. + +The ``nova-compute-ironic`` service instances may be configured +individually via the existing override mechanism. For example, the +creation of ``nova/nova-compute-ironic-1.conf`` will allow variables +for that specific service to be overridden. + OpenStack policy customisation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/kolla_ansible/nova_filters.py b/kolla_ansible/nova_filters.py index 4bb5cbdf6..f9be3eb98 100644 --- a/kolla_ansible/nova_filters.py +++ b/kolla_ansible/nova_filters.py @@ -15,6 +15,8 @@ import jinja2 import re +from kolla_ansible import exception + def extract_cell(list_cells_cli_output, cell_name): """Extract cell settings from nova_manage CLI @@ -85,8 +87,57 @@ def namespace_haproxy_for_cell(services, cell_name): return services +def get_expected_ironic_compute_services(ironic_compute_batch_info, + multi_compute_conf, + nova_cell_compute_ironic_group): + """Return a List of expected Ironic compute services + + :param ironic_compute_batch_info: A dict containing configuration + information about Ironic Compute hosts. + :param multi_compute_conf: Ironic multi-compute config + :param nova_cell_compute_ironic_group: Ansible group name for + nova-compute-ironic. Eg. 'nova-compute-ironic'. + :returns: A list of Nova Compute Ironic service hostnames. + """ + classic_info = ironic_compute_batch_info['classic'] + multi_compute_info = ironic_compute_batch_info['multi'] + if len(multi_compute_conf) == 0: + return [f'{host}-ironic' for host in classic_info] + + expected_services = [] + group_prefix = nova_cell_compute_ironic_group + '-' + for host in multi_compute_info: + iterate_vars = [ + int(group.split(group_prefix)[1]) + for group in host if group_prefix in group] + for i in iterate_vars: + try: + service_conf = multi_compute_conf[i - 1] + except IndexError: + raise exception.FilterError( + "Unable to look up multi-compute Ironic config for " + "{g}-{i} inventory group. Check that " + "nova_multi_compute_ironic_config has an entry for " + "each defined group.".format( + g=nova_cell_compute_ironic_group, + i=i)) + expected_services.append( + _get_expected_compute_service_name(service_conf)) + return expected_services + + +def _get_expected_compute_service_name(service_conf): + custom_host = service_conf.get('custom_host') + if custom_host: + return f"{custom_host}-ironic" + cg = service_conf.get('conductor_group', 'default') + sk = service_conf.get('shard_key', 'default') + return f"{cg}-{sk}-ironic" + + def get_filters(): return { "extract_cell": extract_cell, "namespace_haproxy_for_cell": namespace_haproxy_for_cell, + "get_expected_ironic_compute_services": get_expected_ironic_compute_services, # noqa } diff --git a/kolla_ansible/tests/unit/test_nova_filters.py b/kolla_ansible/tests/unit/test_nova_filters.py index e8d5c507a..d436b0ef1 100644 --- a/kolla_ansible/tests/unit/test_nova_filters.py +++ b/kolla_ansible/tests/unit/test_nova_filters.py @@ -73,6 +73,92 @@ class TestFilters(unittest.TestCase): self.assertRaisesRegex(jinja2.TemplateRuntimeError, 'duplicates', self._test_extract_cell, test_data, 'cell0001') + def test_get_expected_ironic_compute_services_multi_compute(self): + example_ironic_compute_conf = { + # We expect the classic config to be ignored + 'classic': ['custom-host-nova-compute'], + # We have a batch of 2 Ironic compute hosts which should + # run a total of 3 Nova compute Ironic instances + 'multi': [ + ['nova-compute-ironic-1', + 'nova-compute-ironic-2', + 'nova-compute-ironic', + 'something'], + ['nova-compute-ironic-3', + 'nova-compute-ironic', + 'something'], + ] + } + multi_conf = [ + {"custom_host": "some_custom_host"}, + {"shard_key": "shard_1", "conductor_group": "location_1"}, + {"shard_key": "shard_2", "conductor_group": "location_1"}, + {"conductor_group": "location_2"}, + {"shard_key": "shard_1"} + ] + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf, multi_conf, 'nova-compute-ironic') + expected = [ + 'some_custom_host-ironic', + 'location_1-shard_1-ironic', + 'location_1-shard_2-ironic', + ] + self.assertListEqual(actual, expected) + + def test_get_expected_ironic_compute_services_no_compute(self): + example_ironic_compute_conf = { + 'classic': [], + 'multi': [] + } + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf, [], 'nova-compute-ironic') + expected = [] + self.assertListEqual(actual, expected) + + def test_get_expected_ironic_compute_services_classic_compute(self): + example_ironic_compute_conf = { + 'classic': ['custom-foo'], + 'multi': [] + } + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf, [], 'nova-compute-ironic') + expected = ['custom-foo-ironic'] + self.assertListEqual(actual, expected) + + def test_get_expected_ironic_compute_services_classic_compute_multi(self): + # NOTE(dougszu): This isn't a recommended configuration + example_ironic_compute_conf = { + 'classic': ['custom-foo', 'ctrl02', 'ctrl03'], + 'multi': [] + } + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf, [], 'nova-compute-ironic') + expected = [ + 'custom-foo-ironic', + 'ctrl02-ironic', + 'ctrl03-ironic', + ] + self.assertListEqual(actual, expected) + + def test_get_expected_ironic_compute_services_bad_config(self): + example_ironic_compute_conf = { + 'classic': [], + 'multi': [ + ['my-custom-group-1', + 'my-custom-group-2', + 'my-custom-group', + 'something'], + ] + } + multi_conf = [ + {"shard_key": "shard_1", "conductor_group": "location_1"}, + ] + with self.assertRaisesRegex(Exception, + 'Unable to look up multi-compute Ironic ' + 'config for my-custom-group-2'): + filters.get_expected_ironic_compute_services( + example_ironic_compute_conf, multi_conf, 'my-custom-group') + def test_namespace_haproxy_for_cell_with_empty_name(self): example_services = { 'nova-novncproxy': { diff --git a/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml new file mode 100644 index 000000000..c99523feb --- /dev/null +++ b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds support for deploying multiple instances of the Nova Compute Ironic + service on the same host. This is useful in large baremetal deployments. +upgrade: + - | + If you are migrating to the new feature to deploy multiple instances of + Nova Compute Ironic, you are first required to manually remove the + existing Nova Compute Ironic Docker container, and config directories. diff --git a/roles/openstack-clients/defaults/main.yml b/roles/openstack-clients/defaults/main.yml index e1f53e279..034618843 100644 --- a/roles/openstack-clients/defaults/main.yml +++ b/roles/openstack-clients/defaults/main.yml @@ -11,7 +11,7 @@ openstack_clients_pip_packages: - package: python-heatclient enabled: true - package: python-ironicclient - enabled: "{{ scenario == 'ironic' }}" + enabled: "{{ scenario in ['ironic', 'multi-compute-ironic'] }}" - package: python-magnumclient enabled: "{{ scenario == 'magnum' }}" - package: python-neutronclient diff --git a/tests/run.yml b/tests/run.yml index 8dd6c62c6..efa817e24 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -208,10 +208,10 @@ # ironic.conf - src: "tests/templates/ironic-overrides.j2" dest: /etc/kolla/config/ironic.conf - when: "{{ scenario == 'ironic' }}" + when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}" - src: "tests/templates/tenks-deploy-config.yml.j2" dest: "{{ ansible_env.HOME }}/tenks.yml" - when: "{{ scenario == 'ironic' }}" + when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}" # keystone-federation config - src: "tests/templates/keystone-federation/idp.example.org%2Frealms%2Fexample.conf.j2" dest: "/etc/kolla/config/keystone-httpd/federation/oidc/metadata/idp.example.org%2Frealms%2Fexample.conf" @@ -243,7 +243,7 @@ dest: ironic-agent.initramfs - src: "ipa-centos9-{{ zuul.branch | replace('/', '-') }}.kernel" dest: ironic-agent.kernel - when: scenario == "ironic" + when: scenario == "ironic" or scenario == "multi-compute-ironic" - block: - name: Slurp requirements.yml @@ -427,7 +427,7 @@ EXT_NET_GATEWAY: "{{ neutron_external_network_prefix }}1" EXT_NET_DEMO_ROUTER_ADDR: "{{ neutron_external_network_prefix }}10" SCENARIO: "{{ scenario }}" - when: openstack_core_tested or scenario in ['ironic', 'magnum', 'nfv', 'zun', 'octavia'] + when: openstack_core_tested or scenario in ['ironic', 'multi-compute-ironic' ,'magnum', 'nfv', 'zun', 'octavia'] - name: Test OVN VPNaaS import_role: @@ -494,7 +494,8 @@ chdir: "{{ kolla_ansible_src_dir }}" environment: TLS_ENABLED: "{{ tls_enabled }}" - when: scenario == "ironic" + MULTI_COMPUTE_IRONIC: "{{ scenario == 'multi-compute-ironic' }}" + when: scenario == "ironic" or scenario == "multi-compute-ironic" - name: Run test-magnum.sh script script: diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index f5f005acd..ca2ad562e 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -138,6 +138,22 @@ enable_proxysql: "yes" enable_prometheus_proxysql_exporter: "yes" {% endif %} +{% if scenario == "multi-compute-ironic" %} +nova_multi_compute_ironic_config: + - {} + - "shard_key": "shard_1" + "conductor_group": "location_1" + - "shard_key": "shard_2" + "conductor_group": "location_1" + - "conductor_group": "location_2" + +enable_ironic: "yes" +enable_ironic_pxe_filter: "yes" +ironic_dnsmasq_dhcp_ranges: + - range: "10.42.0.2,10.42.0.254,255.255.255.0" +ironic_internal_endpoint: http://192.0.2.1:6385 +{% endif %} + {% if scenario == "mariadb" %} enable_fluentd: "yes" enable_mariadb: "yes" diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 2991518ce..a3bea7213 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -6,7 +6,7 @@ # primary and secondary while the parent group (control) uses only primary. [control] -{% if scenario == 'masakari' %} +{% if scenario in ['masakari', 'multi-compute-ironic'] %} {% for host in hostvars if host in ['primary'] %} {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} {% endfor %} @@ -299,8 +299,25 @@ nova [nova-spicehtml5proxy:children] nova +{% if scenario == 'multi-compute-ironic' %} +[nova-compute-ironic-1] +primary +[nova-compute-ironic-2] +secondary1 +[nova-compute-ironic-3] +secondary2 +[nova-compute-ironic-4] +secondary2 + +[nova-compute-ironic:children] +nova-compute-ironic-1 +nova-compute-ironic-2 +nova-compute-ironic-3 +nova-compute-ironic-4 +{% else %} [nova-compute-ironic:children] nova +{% endif %} [nova-serialproxy:children] nova diff --git a/tests/templates/ironic-overrides.j2 b/tests/templates/ironic-overrides.j2 index be944b307..9603d6b2c 100644 --- a/tests/templates/ironic-overrides.j2 +++ b/tests/templates/ironic-overrides.j2 @@ -2,6 +2,16 @@ enabled_inspect_interfaces = no-inspect, agent default_inspect_interface = agent +{% if scenario == "multi-compute-ironic" %} +{% raw %} +{% set cg = 'location_1' if inventory_hostname == 'secondary1' else 'location_2' if inventory_hostname == 'secondary2' %} +{% if cg %} +[conductor] +conductor_group = {{ cg }} +{% endif %} +{% endraw %} +{% endif %} + [neutron] cleaning_network = public1 inspection_network = public1 diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 0758a8067..7cb2ec76f 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -4,6 +4,7 @@ templates: - ansible-role-jobs - check-requirements + - kolla-ansible-scenario-multi-compute-ironic - kolla-ansible-scenario-aio - kolla-ansible-scenario-bifrost - kolla-ansible-scenario-cells diff --git a/zuul.d/scenarios/multi-compute-ironic.yaml b/zuul.d/scenarios/multi-compute-ironic.yaml new file mode 100644 index 000000000..814116a1f --- /dev/null +++ b/zuul.d/scenarios/multi-compute-ironic.yaml @@ -0,0 +1,47 @@ +--- +- job: + name: kolla-ansible-multi-compute-ironic-base + parent: kolla-ansible-base + voting: false + files: !inherit + - ^ansible/group_vars/all/(nova|ironic).yml + - ^ansible/roles/(nova|nova-cell|ironic)/ + - ^tests/deploy-tenks\.sh$ + - ^tests/templates/ironic-overrides\.j2$ + - ^tests/templates/tenks-deploy-config\.yml\.j2$ + - ^tests/test-dashboard\.sh$ + - ^tests/test-ironic\.sh$ + required-projects: + - openstack/tenks + vars: + scenario: multi-compute-ironic + scenario_images_extra: + - ^dnsmasq + - ^ironic + - ^iscsid + tls_enabled: false + +- job: + name: kolla-ansible-debian-trixie-multi-compute-ironic + parent: kolla-ansible-multi-compute-ironic-base + nodeset: kolla-ansible-debian-trixie-multi-16GB + +- job: + name: kolla-ansible-rocky-10-multi-compute-ironic + parent: kolla-ansible-multi-compute-ironic-base + nodeset: kolla-ansible-rocky-10-multi-16GB + +- job: + name: kolla-ansible-ubuntu-noble-multi-compute-ironic + parent: kolla-ansible-multi-compute-ironic-base + nodeset: kolla-ansible-ubuntu-noble-multi-16GB + +- project-template: + name: kolla-ansible-scenario-multi-compute-ironic + description: | + Runs Kolla-Ansible Nova Multi Compute Ironic scenario jobs. + check: + jobs: + - kolla-ansible-debian-trixie-multi-compute-ironic + - kolla-ansible-rocky-10-multi-compute-ironic + - kolla-ansible-ubuntu-noble-multi-compute-ironic |