Ensure OpenSearch waits for recovery during upgrade

Change-Id: I8825f41a1753eb323e0fdf1bf38a679670e01ba7
Signed-off-by: Alex Welsh <alex@stackhpc.com>
(cherry picked from commit 37194fcab7883b8626f2eab5a4f6a19246b85ec9)

3 files changed, 52 insertions, 1 deletions

diff --git a/ansible/library/kolla_toolbox.py b/ansible/library/kolla_toolbox.py
index faa67dbe9..3e5792021 100644
--- a/ansible/library/kolla_toolbox.py
+++ b/ansible/library/kolla_toolbox.py
@@ -470,7 +470,14 @@ class KollaAnsibleModule(AnsibleModule):
     visible for debugging.
     """
 
-    _NO_LOG_KEYS = frozenset({'auth', 'login_password', 'password'})
+    _NO_LOG_KEYS = frozenset(
+        {
+            'auth',
+            'login_password',
+            'password',
+            'url_password',
+        }
+    )
 
     def _log_invocation(self):
         for param in ('module_args', 'module_extra_vars'):
diff --git a/ansible/roles/opensearch/handlers/main.yml b/ansible/roles/opensearch/handlers/main.yml
index 0a279a9b0..361d9fcf2 100644
--- a/ansible/roles/opensearch/handlers/main.yml
+++ b/ansible/roles/opensearch/handlers/main.yml
@@ -57,6 +57,29 @@
     dimensions: "{{ service.dimensions }}"
     healthcheck: "{{ service.healthcheck | default(omit) }}"
 
+- name: Wait for OpenSearch cluster to recover
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_internal_endpoint }}/_cluster/health?wait_for_status=yellow&timeout=30s"
+      timeout: 35
+      status_code: 200
+      validate_certs: "{{ kolla_enable_tls_internal | bool }}"
+      return_content: true
+      body_format: json
+      ca_path: "{{ openstack_cacert }}"
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+  retries: 60
+  delay: 5
+  register: result
+  until: result.get('status') == 200
+  listen: "Restart opensearch container"
+  when:
+    - kolla_action != "config"
+
 - name: Restart opensearch-dashboards container
   vars:
     service_name: "opensearch-dashboards"
diff --git a/ansible/roles/opensearch/tasks/post-config.yml b/ansible/roles/opensearch/tasks/post-config.yml
index 290e88365..deef86b9f 100644
--- a/ansible/roles/opensearch/tasks/post-config.yml
+++ b/ansible/roles/opensearch/tasks/post-config.yml
@@ -88,6 +88,27 @@
     - not ansible_check_mode
     - opensearch_retention_policy_check.status == 404
 
+- name: Wait for OpenSearch Dashboards API to become ready
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_dashboards_internal_endpoint }}/api/status"
+      method: GET
+      status_code: 200
+      url_username: "{{ opensearch_dashboards_user }}"
+      url_password: "{{ opensearch_dashboards_password }}"
+      force_basic_auth: true
+      ca_path: "{{ openstack_cacert }}"
+  register: result
+  until: result.get('status') == 200
+  retries: 30
+  delay: 10
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+  when: not ansible_check_mode
+
 - name: Create OpenSearch Dashboards Index Pattern
   become: true
   kolla_toolbox: