diff options
| -rw-r--r-- | ansible/roles/nova/defaults/main.yml | 6 | ||||
| -rw-r--r-- | etc/kolla/globals.yml | 15 | ||||
| -rw-r--r-- | releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml | 13 |
3 files changed, 33 insertions, 1 deletions
diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml index 215826cdd..020048781 100644 --- a/ansible/roles/nova/defaults/main.yml +++ b/ansible/roles/nova/defaults/main.yml @@ -168,7 +168,11 @@ nova_metadata_healthcheck: nova_scheduler_enable_healthchecks: "{{ enable_container_healthchecks }}" nova_scheduler_healthcheck_interval: "{{ default_container_healthcheck_interval }}" -nova_scheduler_healthcheck_retries: "{{ default_container_healthcheck_retries }}" + +# NOTE(mikal): this one is deliberately overridden because of the issues discussed +# in releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml +nova_scheduler_healthcheck_retries: 8 + nova_scheduler_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}" nova_scheduler_healthcheck_test: ["CMD-SHELL", "healthcheck_port nova-scheduler {{ om_rpc_port }}"] nova_scheduler_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}" diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 2d7cdcda4..f5f8cdad2 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -219,6 +219,21 @@ workaround_ansible_issue_8743: true #default_container_healthcheck_timeout: 30 #default_container_healthcheck_retries: 3 #default_container_healthcheck_start_period: 5 +# Individual services accept per-service overrides of the options above, +# named <service>_healthcheck_<option>. nova_scheduler ships with a wider +# one: its healthcheck asserts an established AMQP connection, but every +# deploy and every reconfigure signals the scheduler to restart its +# workers, and they take [DEFAULT]/manager_shutdown_timeout in nova.conf +# (160 seconds by default) to come back. The default 30 second interval +# with 3 retries only covers 90 seconds of that. +#nova_scheduler_healthcheck_retries: 8 +# The conductors wait for the same period, but are only ever stopped by +# the container engine, which kills them after docker_graceful_timeout +# and so stays inside what the default healthcheck covers. Raise these +# to match if docker_graceful_timeout is raised towards +# manager_shutdown_timeout. +#nova_conductor_healthcheck_retries: 3 +#nova_super_conductor_healthcheck_retries: 3 ################## # Firewall options diff --git a/releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml b/releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml new file mode 100644 index 000000000..2cf4e2e8c --- /dev/null +++ b/releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml @@ -0,0 +1,13 @@ +--- +fixes: + - | + The ``nova_scheduler`` container healthcheck asserts an established AMQP + connection, but every deploy and every reconfigure signals the scheduler + to restart its workers and they take + ``[DEFAULT]/manager_shutdown_timeout`` (160 seconds by default) to come + back. The default 30 second interval with 3 retries only covers 90 + seconds of that, so the container could be reported unhealthy for the + rest of the restart even though the scheduler was fine. + ``nova_scheduler_healthcheck_retries`` now defaults to 8, which covers + the whole window, and is documented in the sample ``globals.yml``. + `LP#2162860 <https://bugs.launchpad.net/kolla/+bug/2162860>`__ |