Ensure OpenSearch waits for recovery during upgrade

There's actually two fixes here

The first waits until the OpenSearch state is healthy before moving on
to Dashboards. Otherwise, OpenSearch Dashboards does its internal
`.kibana` index migration before OpenSearch has fully recovered. This
creates a corrupted `.kibana_1` index, making Dashboard enter an
infinite polling loop and fail to start.

The second adds retries to the "Create OpenSearch Dashboards Index
Pattern" task. Dashboards seems to take a while to start up
post-upgrade, I think again, it's doing some internal migrations for the
latest version. Without first checking the API is ready, it consistently
fails to be ready, and throws 503s by the time we try to create index
patterns.

Change-Id: Ic06ccc6b7ec22fa1b0cfa2a144878672d314e93e
Closes-Bug: #2161333
Signed-off-by: Alex Welsh <alex@stackhpc.com>
(cherry picked from commit 79f6f2a049bad1d3cb51ba50981a01f63503a329)

3 files changed, 50 insertions, 0 deletions

diff --git a/ansible/roles/opensearch/handlers/main.yml b/ansible/roles/opensearch/handlers/main.yml
index 0a279a9b0..361d9fcf2 100644
--- a/ansible/roles/opensearch/handlers/main.yml
+++ b/ansible/roles/opensearch/handlers/main.yml
@@ -57,6 +57,29 @@
     dimensions: "{{ service.dimensions }}"
     healthcheck: "{{ service.healthcheck | default(omit) }}"
 
+- name: Wait for OpenSearch cluster to recover
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_internal_endpoint }}/_cluster/health?wait_for_status=yellow&timeout=30s"
+      timeout: 35
+      status_code: 200
+      validate_certs: "{{ kolla_enable_tls_internal | bool }}"
+      return_content: true
+      body_format: json
+      ca_path: "{{ openstack_cacert }}"
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+  retries: 60
+  delay: 5
+  register: result
+  until: result.get('status') == 200
+  listen: "Restart opensearch container"
+  when:
+    - kolla_action != "config"
+
 - name: Restart opensearch-dashboards container
   vars:
     service_name: "opensearch-dashboards"
diff --git a/ansible/roles/opensearch/tasks/post-config.yml b/ansible/roles/opensearch/tasks/post-config.yml
index 290e88365..deef86b9f 100644
--- a/ansible/roles/opensearch/tasks/post-config.yml
+++ b/ansible/roles/opensearch/tasks/post-config.yml
@@ -88,6 +88,27 @@
     - not ansible_check_mode
     - opensearch_retention_policy_check.status == 404
 
+- name: Wait for OpenSearch Dashboards API to become ready
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_dashboards_internal_endpoint }}/api/status"
+      method: GET
+      status_code: 200
+      url_username: "{{ opensearch_dashboards_user }}"
+      url_password: "{{ opensearch_dashboards_password }}"
+      force_basic_auth: true
+      ca_path: "{{ openstack_cacert }}"
+  register: result
+  until: result.get('status') == 200
+  retries: 30
+  delay: 10
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+  when: not ansible_check_mode
+
 - name: Create OpenSearch Dashboards Index Pattern
   become: true
   kolla_toolbox:
diff --git a/releasenotes/notes/bug-2161333-aeaa498fccaadf51.yaml b/releasenotes/notes/bug-2161333-aeaa498fccaadf51.yaml
new file mode 100644
index 000000000..226c5a8eb
--- /dev/null
+++ b/releasenotes/notes/bug-2161333-aeaa498fccaadf51.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes OpenSearch 503 errors on Create OpenSearch Dashboards Index Pattern
+    task during service upgrade.
+    `LP#2161333 <https://launchpad.net/bugs/2161333>`__