From 5e5a2dca0900e5c167e1ab55ef6dc14198ce2903 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 7 Dec 2022 14:50:21 +0000 Subject: Set a log retention policy for OpenSearch We previously used ElasticSearch Curator for managing log retention. Now that we have moved to OpenSearch, we can use the Index State Management (ISM) plugin which is bundled with OpenSearch. This change adds support for automating the configuration of the ISM plugin via the OpenSearch API. By default, it has similar behaviour to the previous ElasticSearch Curator default policy. Closes-Bug: #2047037 Change-Id: I5c6d938f2bc380f1575ee4f16fe17c6dca37dcba --- ansible/roles/opensearch/defaults/main.yml | 48 +++++++++++++++++++ ansible/roles/opensearch/tasks/deploy.yml | 3 ++ ansible/roles/opensearch/tasks/post-config.yml | 65 ++++++++++++++++++++++++++ ansible/roles/opensearch/tasks/upgrade.yml | 3 ++ 4 files changed, 119 insertions(+) create mode 100644 ansible/roles/opensearch/tasks/post-config.yml (limited to 'ansible') diff --git a/ansible/roles/opensearch/defaults/main.yml b/ansible/roles/opensearch/defaults/main.yml index 13bc4248d..da66a9d31 100644 --- a/ansible/roles/opensearch/defaults/main.yml +++ b/ansible/roles/opensearch/defaults/main.yml @@ -58,6 +58,54 @@ opensearch_cluster_name: "kolla_logging" opensearch_heap_size: "1g" opensearch_java_opts: "{% if opensearch_heap_size %}-Xms{{ opensearch_heap_size }} -Xmx{{ opensearch_heap_size }}{% endif %} -Dlog4j2.formatMsgNoLookups=true" +opensearch_apply_log_retention_policy: true + +# Duration after which an index is staged for deletion. This is implemented +# by closing the index. Whilst in this state the index contributes negligible +# load on the cluster and may be manually re-opened if required. +# NOTE: We carry over legacy settings from ElasticSearch Curator if they +# are set. This may be removed in a later release. +opensearch_soft_retention_period_days: "{{ elasticsearch_curator_soft_retention_period_days | default(30) }}" + +# Duration after which an index is permanently erased from the cluster. +opensearch_hard_retention_period_days: "{{ elasticsearch_curator_hard_retention_period_days | default(60) }}" + +opensearch_retention_policy: | + policy: + description: Retention policy for OpenStack logs + error_notification: + default_state: open + states: + - name: open + actions: [] + transitions: + - state_name: close + conditions: + min_index_age: "{{ opensearch_soft_retention_period_days }}d" + - name: close + actions: + - retry: + count: 3 + backoff: exponential + delay: 1m + close: {} + transitions: + - state_name: delete + conditions: + min_index_age: "{{ opensearch_hard_retention_period_days }}d" + - name: delete + actions: + - retry: + count: 3 + backoff: exponential + delay: 1m + delete: {} + transitions: [] + ism_template: + - index_patterns: + - "{{ opensearch_log_index_prefix }}-*" + priority: 1 + #################### # Keystone #################### diff --git a/ansible/roles/opensearch/tasks/deploy.yml b/ansible/roles/opensearch/tasks/deploy.yml index ee17effc6..a0ebfaf7d 100644 --- a/ansible/roles/opensearch/tasks/deploy.yml +++ b/ansible/roles/opensearch/tasks/deploy.yml @@ -10,3 +10,6 @@ - name: Flush handlers meta: flush_handlers + +- include_tasks: post-config.yml + when: opensearch_apply_log_retention_policy | bool diff --git a/ansible/roles/opensearch/tasks/post-config.yml b/ansible/roles/opensearch/tasks/post-config.yml new file mode 100644 index 000000000..ac2605244 --- /dev/null +++ b/ansible/roles/opensearch/tasks/post-config.yml @@ -0,0 +1,65 @@ +--- +- name: Wait for OpenSearch to become ready + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_cluster/stats" + status_code: 200 + register: result + until: result.get('status') == 200 + retries: 30 + delay: 2 + run_once: true + +- name: Check if a log retention policy exists + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention" + method: GET + status_code: 200, 404 + return_content: yes + register: opensearch_retention_policy_check + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + +- name: Create new log retention policy + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention" + method: PUT + status_code: 201 + return_content: yes + body: "{{ opensearch_retention_policy | from_yaml | to_json }}" + body_format: json + register: opensearch_retention_policy_create + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + changed_when: opensearch_retention_policy_create.status == 201 + when: opensearch_retention_policy_check.status == 404 + +- name: Apply retention policy to existing indicies + become: true + vars: + opensearch_set_policy_body: {"policy_id": "retention"} + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/add/{{ opensearch_log_index_prefix }}-*" + method: POST + status_code: 200 + return_content: yes + body: "{{ opensearch_set_policy_body | to_json }}" + body_format: json + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + changed_when: opensearch_retention_policy_create.status == 201 + when: opensearch_retention_policy_check.status == 404 diff --git a/ansible/roles/opensearch/tasks/upgrade.yml b/ansible/roles/opensearch/tasks/upgrade.yml index 2891b64e0..da343e8b7 100644 --- a/ansible/roles/opensearch/tasks/upgrade.yml +++ b/ansible/roles/opensearch/tasks/upgrade.yml @@ -46,3 +46,6 @@ - name: Flush handlers meta: flush_handlers + +- include_tasks: post-config.yml + when: opensearch_apply_log_retention_policy | bool -- cgit v1.2.3