diff options
13 files changed, 144 insertions, 116 deletions
diff --git a/ansible/monasca_cleanup.yml b/ansible/monasca_cleanup.yml new file mode 100644 index 000000000..25b1243a9 --- /dev/null +++ b/ansible/monasca_cleanup.yml @@ -0,0 +1,14 @@ +--- +- name: Cleanup unused Monasca services + hosts: + - monasca-api + - monasca-grafana + - monasca-log-transformer + - monasca-log-persister + - monasca-log-metrics + - monasca-thresh + - monasca-notification + - monasca-persister + roles: + - { role: monasca, + tags: monasca } diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 5a4d506bb..ab219b832 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -20,10 +20,12 @@ monasca_services: mode: "http" external: true port: "{{ monasca_api_port }}" + # NOTE(dszumski): We can remove log_transformer and all other references after the + # Wallaby release monasca-log-transformer: container_name: monasca_log_transformer group: monasca-log-transformer - enabled: true + enabled: false image: "{{ monasca_logstash_image_full }}" volumes: "{{ monasca_log_transformer_default_volumes + monasca_log_transformer_extra_volumes }}" dimensions: "{{ monasca_log_transformer_dimensions }}" @@ -175,7 +177,7 @@ monasca_all_topics: - name: "{{ monasca_transformed_logs_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: False - name: "{{ monasca_events_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index 7eb616282..1346e5356 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -14,21 +14,6 @@ when: - kolla_action != "config" -- name: Restart monasca-log-transformer container - vars: - service_name: "monasca-log-transformer" - service: "{{ monasca_services[service_name] }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - name: Restart monasca-log-persister container vars: service_name: "monasca-log-persister" diff --git a/ansible/roles/monasca/tasks/cleanup.yml b/ansible/roles/monasca/tasks/cleanup.yml new file mode 100644 index 000000000..dd23dbc15 --- /dev/null +++ b/ansible/roles/monasca/tasks/cleanup.yml @@ -0,0 +1,22 @@ +--- +- name: Stop and remove containers for disabled monasca services + become: true + kolla_docker: + action: "stop_and_remove_container" + name: "{{ item.value.container_name }}" + when: + - inventory_hostname in groups[item.value.group] + - not item.value.enabled | bool + with_dict: "{{ monasca_services }}" + +- name: Removing config for any disabled services + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "absent" + become: true + when: + - inventory_hostname in groups[item.value.group] + - not item.value.enabled | bool + with_dict: "{{ monasca_services }}" + +# NOTE(dszumski): Docker volume removal is currently a manual procedure diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index ed1f6919b..41f3a0869 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -160,29 +160,11 @@ notify: - Restart monasca-api container -- name: Copying over monasca-log-transformer config - vars: - service: "{{ monasca_services['monasca-log-transformer'] }}" - template: - src: "{{ item }}" - dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf" - mode: "0660" - become: true - with_first_found: - - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf" - - "{{ node_custom_config }}/monasca/log-transformer.conf" - - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2" - when: - - inventory_hostname in groups[service['group']] - - service.enabled | bool - notify: - - Restart monasca-log-transformer container - - name: Ensuring logstash patterns folder exists vars: - service: "{{ monasca_services['monasca-log-transformer'] }}" + service: "{{ monasca_services['monasca-log-persister'] }}" file: - path: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns" + path: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns" state: "directory" mode: "0770" become: true @@ -200,10 +182,10 @@ - name: Copying over custom logstash patterns vars: - service: "{{ monasca_services['monasca-log-transformer'] }}" + service: "{{ monasca_services['monasca-log-persister'] }}" template: src: "{{ item.path }}" - dest: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns/{{ item.path | basename }}" + dest: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns/{{ item.path | basename }}" mode: "0660" with_items: "{{ monasca_custom_logstash_patterns.files }}" become: true @@ -211,7 +193,7 @@ - inventory_hostname in groups[service['group']] - service.enabled | bool notify: - - Restart monasca-log-transformer container + - Restart monasca-log-persister container - name: Copying over monasca-log-persister config vars: diff --git a/ansible/roles/monasca/tasks/upgrade.yml b/ansible/roles/monasca/tasks/upgrade.yml index d42398e4a..c5b8e5bbe 100644 --- a/ansible/roles/monasca/tasks/upgrade.yml +++ b/ansible/roles/monasca/tasks/upgrade.yml @@ -14,6 +14,8 @@ - import_tasks: config.yml +- import_tasks: cleanup.yml + - import_tasks: check-containers.yml # NOTE(dszumski): We don't want old Grafana instances running after diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 index ad651a20c..4ab974be4 100644 --- a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 @@ -10,7 +10,7 @@ input { kafka { bootstrap_servers => "{{ monasca_kafka_servers }}" - topics => ["{{ monasca_transformed_logs_topic }}"] + topics => ["{{ monasca_raw_logs_topic }}"] group_id => "log_metrics" consumer_threads => "{{ monasca_log_pipeline_threads }}" codec => json diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 index 732c07e5a..c4e699477 100644 --- a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 @@ -1,15 +1,41 @@ -# Persist transformed logs to Elasticsearch +# Persist logs to Elasticsearch. input { kafka { bootstrap_servers => "{{ monasca_kafka_servers }}" - topics => ["{{ monasca_transformed_logs_topic }}"] + topics => ["{{ monasca_raw_logs_topic }}"] group_id => "log_persister" consumer_threads => "{{ monasca_log_pipeline_threads }}" codec => json } } +filter { + # Update the timestamp of the event based on the time in the message. + date { + match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"] + remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] + } + + # Monasca Log API adds a timestamp when it processes a log entry. This + # timestamp needs to be converted from seconds since the epoch for + # Elasticsearch to parse it correctly. Here we make that conversion. + date { + match => ["creation_time", "UNIX"] + target => "creation_time" + } + + # OpenStack log levels are uppercase, and syslog are lowercase. + # Furthermore, syslog has more log levels that OpenStack. To avoid + # mapping syslog log levels to OpenStack log levels, we standardise + # on the syslog style here. + if [log][dimensions][log_level] { + mutate { + lowercase => [ "[log][dimensions][log_level]" ] + } + } +} + output { elasticsearch { index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}" diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 deleted file mode 100644 index b63792537..000000000 --- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 +++ /dev/null @@ -1,46 +0,0 @@ -# Provide input/output streams for transforming Monasca logs. -# Filters should be provided in other configuration files. - -input { - kafka { - bootstrap_servers => "{{ monasca_kafka_servers }}" - topics => ["{{ monasca_raw_logs_topic }}"] - group_id => "log_transformer" - consumer_threads => "{{ monasca_log_pipeline_threads }}" - codec => json - } -} - -filter { - # Update the timestamp of the event based on the time in the message. - date { - match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"] - remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] - } - - # Monasca Log API adds a timestamp when it processes a log entry. This - # timestamp needs to be converted from seconds since the epoch for - # Elasticsearch to parse it correctly. Here we make that conversion. - date { - match => ["creation_time", "UNIX"] - target => "creation_time" - } - - # OpenStack log levels are uppercase, and syslog are lowercase. - # Furthermore, syslog has more log levels that OpenStack. To avoid - # mapping syslog log levels to OpenStack log levels, we standardise - # on the syslog style here. - if [log][dimensions][log_level] { - mutate { - lowercase => [ "[log][dimensions][log_level]" ] - } - } -} - -output { - kafka { - codec => json - bootstrap_servers => "{{ monasca_kafka_servers }}" - topic_id => "{{ monasca_transformed_logs_topic }}" - } -} diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 deleted file mode 100644 index 6d5cee36e..000000000 --- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 +++ /dev/null @@ -1,25 +0,0 @@ -{ - "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf", - "config_files": [ - { - "source": "{{ container_config_directory }}/log-transformer.conf", - "dest": "/etc/logstash/conf.d/log-transformer.conf", - "owner": "logstash", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/logstash_patterns/*", - "dest": "/etc/logstash/conf.d/patterns/", - "owner": "logstash", - "perm": "0600", - "optional": true - } - ], - "permissions": [ - { - "path": "/var/log/kolla/logstash", - "owner": "logstash:kolla", - "recurse": true - } - ] -} diff --git a/doc/source/reference/logging-and-monitoring/monasca-guide.rst b/doc/source/reference/logging-and-monitoring/monasca-guide.rst index a73d4a425..afd876bea 100644 --- a/doc/source/reference/logging-and-monitoring/monasca-guide.rst +++ b/doc/source/reference/logging-and-monitoring/monasca-guide.rst @@ -318,6 +318,42 @@ Apply the password changes by running the following command: kolla-ansible reconfigure -t monasca +Cleanup +~~~~~~~ + +From time-to-time it may be necessary to manually invoke the Monasca cleanup +command. If this is required during an upgrade it will be mentioned in the +release notes. It may also be necessary to run the cleanup command when +disabling certain parts of the Monasca pipeline. A full list of scenarios in +which you must run the cleanup command is given below: + +- Upgrading from Victoria to Wallaby to remove the unused Monasca Log + Transformer service + +The cleanup command can be invoked from the Kolla Ansible CLI, for example: + +.. code-block:: console + + kolla-ansible monasca_cleanup + +Following cleanup, you may also choose to remove unused container volumes. +It is recommended to run this manually on each Monasca service host. Note +that `docker prune` will indiscriminately remove all unused volumes, +which may not always be what you want. If you wish to keep a subset of +unused volumes, you can remove them individually. + +To remove all unused volumes on a host: + +.. code-block:: console + + docker prune + +To remove a single unused volume, run for example: + +.. code-block:: console + + docker volume rm monasca_log_transformer_data + System requirements and performance impact ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -335,9 +371,8 @@ Monasca will deploy the following Docker containers: * Monasca Agent Statsd * Monasca API * Monasca Log API -* Monasca Log Transformer (Logstash) * Monasca Log Metrics (Logstash) -* Monasca Log Perister (Logstash) +* Monasca Log Persister (Logstash) * Monasca Notification * Monasca Persister * Monasca Thresh (Apache Storm topology) diff --git a/releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml b/releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml new file mode 100644 index 000000000..4770a85e3 --- /dev/null +++ b/releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml @@ -0,0 +1,24 @@ +--- +upgrade: + - | + Monasca Log Transformer has been merged with Monasca Log Persister to + improve performance and reduce resource consumption. Any custom Monasca + Log Transformer configuration should be either merged into Monasca Log + Persister configuration, or moved outside of the Monasca pipeline, for + example, to Fluentd. Any custom Monasca Log Metrics config will also + need to be updated to read from the raw logs pipeline, rather than the + transformed logs pipeline. The transformed logs pipeline will be + removed from Kafka automatically, as will any log transformer containers. + There will be a short interruption to logging services whilst the + pipeline is updated. During this time it's likely that a small window + of logs will be lost from the transformed logs Kafka queue. If this + is a problem, the Monasca API should be stopped on all nodes prior to + upgrading Monasca. This will allow the transformed logs topic to drain + into Elasticsearch before the pipeline is reconfigured. Services such + as Fluentd, which post logs to the Monasca API, should buffer logs + whilst this happens up to the maximum configured buffer. Note that + there may be other services forwarding logs, and these will need to be + inspected independently. The Log Transformer volumes will remain on + the monitoring nodes and can be manually removed as described in the + `documentation + <https://docs.openstack.org/kolla-ansible/latest/reference/logging-and-monitoring/monasca-guide.html#cleanup>`__. diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 56e675e8a..bca3869f0 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -133,6 +133,7 @@ Commands: mariadb_backup Take a backup of MariaDB databases --full (default) --incremental + monasca_cleanup Remove unused containers for the Monasca service bootstrap-servers Bootstrap servers with kolla deploy dependencies destroy Destroy Kolla containers, volumes and host configuration --include-images to also destroy Kolla images @@ -175,6 +176,7 @@ prechecks check mariadb_recovery mariadb_backup +monasca_cleanup bootstrap-servers destroy deploy @@ -359,6 +361,11 @@ case "$1" in EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE}" PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml" ;; +(monasca_cleanup) + ACTION="Cleanup unused Monasca services" + EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=cleanup" + PLAYBOOK="${BASEDIR}/ansible/monasca_cleanup.yml" + ;; (destroy) ACTION="Destroy Kolla containers, volumes and host configuration" PLAYBOOK="${BASEDIR}/ansible/destroy.yml" |