diff options
| -rw-r--r-- | ansible/roles/nova-cell/defaults/main.yml | 11 | ||||
| -rw-r--r-- | ansible/roles/nova-cell/templates/libvirtd.conf.j2 | 10 | ||||
| -rw-r--r-- | doc/source/reference/compute/libvirt-guide.rst | 23 | ||||
| -rw-r--r-- | releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml | 18 |
4 files changed, 62 insertions, 0 deletions
diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml index 790190fb0..918de4463 100644 --- a/ansible/roles/nova-cell/defaults/main.yml +++ b/ansible/roles/nova-cell/defaults/main.yml @@ -639,6 +639,17 @@ libvirt_sasl_authname: "nova" libvirt_sasl_mech_list: - "{{ 'SCRAM-SHA-256' if libvirt_tls | bool else 'DIGEST-MD5' }}" +# Connection processing limits for libvirtd. nova-compute is effectively the +# only privileged client of libvirtd under Kolla, so libvirt's stock per-client +# request cap (max_client_requests=5) is the binding constraint rather than a +# safety net: concurrent instance builds plus the resource-tracker periodic +# routinely exceed five in-flight RPCs, which starves keepalive responses until +# libvirtd drops the connection. Raise the per-client cap, and the worker pool +# along with it so that lifting the per-client limit does not starve other +# connections (live migration, health checks, periodics). +libvirt_max_client_requests: 20 +libvirt_max_workers: 50 + #################### # Kolla #################### diff --git a/ansible/roles/nova-cell/templates/libvirtd.conf.j2 b/ansible/roles/nova-cell/templates/libvirtd.conf.j2 index aa01f743d..d156fb4be 100644 --- a/ansible/roles/nova-cell/templates/libvirtd.conf.j2 +++ b/ansible/roles/nova-cell/templates/libvirtd.conf.j2 @@ -25,3 +25,13 @@ listen_addr = "{{ migration_interface_address }}" # Enable read-only access to libvirt socket auth_unix_ro = "none" {% endif %} + +# Connection processing limits. libvirt's defaults (max_workers=20, +# max_client_requests=5) assume a multi-tenant host with many mutually +# distrustful clients. Under Kolla, nova-compute is effectively the sole +# privileged client, so the per-client cap of 5 becomes the binding +# constraint: concurrent builds plus the resource-tracker periodic exceed +# five in-flight RPCs and starve keepalive responses, causing libvirtd to +# drop the connection after keepalive_interval (5s) x keepalive_count (5). +max_workers = {{ libvirt_max_workers }} +max_client_requests = {{ libvirt_max_client_requests }} diff --git a/doc/source/reference/compute/libvirt-guide.rst b/doc/source/reference/compute/libvirt-guide.rst index 24a55e1fb..d6212b908 100644 --- a/doc/source/reference/compute/libvirt-guide.rst +++ b/doc/source/reference/compute/libvirt-guide.rst @@ -198,3 +198,26 @@ Since the Yoga release, the ``kolla-ansible certificates`` command generates certificates for libvirt TLS. A single key and certificate is used for all hosts, with a Subject Alternative Name (SAN) entry for each compute host hostname. + +Libvirt Connection Limits +========================= + +By default ``libvirt`` is configured by upstream packages with reasonable +defaults for an environment where many users are starting and stopping virtual +machines. This isn't true for Kolla Hypervisor nodes, where a single +``nova-compute`` container manages all of the virtual machine instances on the +machine. + +Kolla-Ansible therefore tunes the following values in ``libvirtd.conf``: + +* libvirt_max_client_requests: raised to 20 compared to the usual default of + 5. +* libvirt_max_workers: raised to 50 compared to the usual default of 20. + +Both of these values may be overridden in ``globals.yml``. + +This means that ``nova-compute`` can therefore perform 20 operations via +libvirt simultaneously instead of the usual default of 5. These values should +be safe, but if you see libvirt error messages saying ``Client hit max requests +limit``, you may want to consider an additional increase -- especially if you +have particularly large hypervisors. diff --git a/releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml b/releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml new file mode 100644 index 000000000..fcc8a65d4 --- /dev/null +++ b/releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + The ``nova_libvirt`` container's ``libvirtd.conf`` now raises libvirt's + connection processing limits above their stock defaults, which are tuned + for a multi-tenant host rather than for Kolla's single-privileged-client + pattern. ``max_client_requests`` is raised from 5 to 20 and ``max_workers`` + from 20 to 50. Both are exposed as the ``libvirt_max_client_requests`` and + ``libvirt_max_workers`` variables so they can be tuned further. +fixes: + - | + Fixed intermittent instance build failures (``Cannot recv data: + Input/output error`` during ``defineXML``, leading to instances in + ``ERROR`` state) that occurred when several instances were built + concurrently. nova-compute is effectively the sole client of ``libvirtd``, + and libvirt's default per-client request cap of 5 was being exceeded by + concurrent builds plus periodic tasks, starving keepalive responses until + ``libvirtd`` dropped the connection. |