From 491aff0b88bc3dac13ea9180c952854fd0a67684 Mon Sep 17 00:00:00 2001
From: Éric Lemoine <elemoine@mirantis.com>
Date: Wed, 24 Feb 2016 16:05:36 +0100
Subject: Make Heka send logs to Elasticsearch

This patch includes changes relative to integrating Heka with
Elasticsearch and Kibana.

The main change is the addition of an Heka ElasticSearchOutput plugin
to make Heka send the logs it collects to Elasticsearch.

Since Logstash is not used the enable_elk deploy variable is renamed
to enable_central_logging.

If enable_central_logging is false then Elasticsearch and Kibana are
not started, and Heka won't attempt to send logs to Elasticsearch.

By default enable_central_logging is set to false. If
enable_central_logging is set to true after deployment then the Heka
container needs to be recreated (for Heka to get the new
configuration).

The Kibana configuration used property names that are deprecated in
Kibana 4.2. This is changed to use non-deprecated property names.

Previously logs read from files and from Syslog had a different Type
in Heka. This is changed to always use "log" for the Type. In this
way just one index instead of two is used in Elasticsearch, making
things easier to the user on the visualization side.

The HAProxy configuration is changed to add entries for Kibana.
Kibana server is now accessible via the internal VIP, and also via
the external VIP if there's one configured.

The HAProxy configuration is changed to add an entry for
Elasticsearch. So Elasticsearch is now accessible via the internal
VIP. Heka uses that channel for communicating with Elasticsearch.

Note that currently the Heka logs include "Plugin
elasticsearch_output" errors when Heka starts. This occurs when Heka
starts processing logs while Elasticsearch is not yet started. These
are transient errors that go away when Elasticsearch is ready. And
with buffering enabled on the ElasticSearchOuput plugin logs will be
buffered and then retransmitted when Elasticsearch is ready.

Change-Id: I6ff7a4f0ad04c4c666e174693a35ff49914280bb
Implements: blueprint central-logging-service
---
 ansible/group_vars/all.yml                         |  4 ++--
 ansible/roles/common/tasks/config.yml              |  8 ++++++++
 .../common/templates/heka-elasticsearch.toml.j2    | 16 +++++++++++++++
 .../roles/common/templates/heka-haproxy.toml.j2    |  4 ++--
 ansible/roles/common/templates/heka-swift.toml.j2  |  2 +-
 ansible/roles/common/templates/heka.json.j2        |  7 +++++++
 ansible/roles/elasticsearch/defaults/main.yml      |  7 +++++--
 .../elasticsearch/templates/elasticsearch.yml.j2   | 15 +++++++++++++-
 ansible/roles/haproxy/templates/haproxy.cfg.j2     | 23 ++++++++++++++++++++++
 ansible/roles/kibana/defaults/main.yml             | 16 ++++-----------
 ansible/roles/kibana/templates/kibana.yml.j2       | 15 +++++++-------
 ansible/site.yml                                   |  4 ++--
 docker/heka/plugins/decoders/os_swift_log.lua      |  2 +-
 docker/heka/plugins/decoders/os_syslog.lua         |  2 +-
 tools/cleanup-containers                           |  7 +++++--
 15 files changed, 98 insertions(+), 34 deletions(-)
 create mode 100644 ansible/roles/common/templates/heka-elasticsearch.toml.j2

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index eca72ff3c..cd1c29205 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -130,7 +130,7 @@ rgw_port: "6780"
 
 mistral_api_port: "8989"
 
-kibana_port: "5601"
+kibana_server_port: "5601"
 
 elasticsearch_port: "9200"
 
@@ -190,7 +190,7 @@ enable_murano: "no"
 enable_ironic: "no"
 enable_magnum: "no"
 enable_mistral: "no"
-enable_elk: "no"
+enable_central_logging: "no"
 enable_mongodb: "no"
 
 ironic_keystone_user: "ironic"
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 18f87c610..8592801ab 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -46,3 +46,11 @@
     - "swift-object-updater"
     - "swift-proxy-server"
     - "swift-rsyncd"
+
+- name: Copying over heka elasticsearch config file
+  template:
+    src: "heka-{{ item }}.toml.j2"
+    dest: "{{ node_config_directory }}/heka/heka-{{ item }}.toml"
+  with_items:
+    - "elasticsearch"
+  when: "{{ enable_central_logging | bool }}"
diff --git a/ansible/roles/common/templates/heka-elasticsearch.toml.j2 b/ansible/roles/common/templates/heka-elasticsearch.toml.j2
new file mode 100644
index 000000000..7b9a351cb
--- /dev/null
+++ b/ansible/roles/common/templates/heka-elasticsearch.toml.j2
@@ -0,0 +1,16 @@
+[elasticsearch_json_encoder]
+type = "ESJsonEncoder"
+index = {{'"%{Type}-%{%Y.%m.%d}"'}}
+es_index_from_timestamp = true
+fields = ["Timestamp", "Type", "Logger", "Severity", "Payload", "Pid", "Hostname", "DynamicFields"]
+
+[elasticsearch_output]
+type = "ElasticSearchOutput"
+server = "{{ internal_protocol }}://{{ kolla_internal_vip_address }}:{{ elasticsearch_port }}"
+message_matcher = "Type == 'log'"
+encoder = "elasticsearch_json_encoder"
+use_buffering = true
+    [elasticsearch_output.buffering]
+    max_buffer_size = 1073741824  # 1024 * 1024 * 1024
+    max_file_size = 134217728  # 128 * 1024 * 1024
+    full_action = "drop"
diff --git a/ansible/roles/common/templates/heka-haproxy.toml.j2 b/ansible/roles/common/templates/heka-haproxy.toml.j2
index d933a6f55..f30dd32d5 100644
--- a/ansible/roles/common/templates/heka-haproxy.toml.j2
+++ b/ansible/roles/common/templates/heka-haproxy.toml.j2
@@ -3,12 +3,12 @@
 
 [haproxy_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:haproxy)/"
+message_matcher = "Fields[programname] =~ /(?i:haproxy)/"
 path = "/var/log/kolla/haproxy/haproxy.log"
 encoder = "syslog_encoder"
 
 [keepalived_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:keepalived)/"
+message_matcher = "Fields[programname] =~ /(?i:keepalived)/"
 path = "/var/log/kolla/haproxy/keepalived.log"
 encoder = "syslog_encoder"
diff --git a/ansible/roles/common/templates/heka-swift.toml.j2 b/ansible/roles/common/templates/heka-swift.toml.j2
index 3bb520e32..8ed4a1e23 100644
--- a/ansible/roles/common/templates/heka-swift.toml.j2
+++ b/ansible/roles/common/templates/heka-swift.toml.j2
@@ -3,6 +3,6 @@
 
 [{{ item }}_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] == '{{ item }}'"
+message_matcher = "Fields[programname] == '{{ item }}'"
 path = "/var/log/kolla/swift/{{ item }}.log"
 encoder = "syslog_encoder"
diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2
index f5bcad297..332790231 100644
--- a/ansible/roles/common/templates/heka.json.j2
+++ b/ansible/roles/common/templates/heka.json.j2
@@ -2,6 +2,13 @@
 {
     "command": "/usr/bin/hekad -config=/etc/heka/",
     "config_files": [
+        {
+            "source": "{{ container_config_directory }}/heka-elasticsearch.toml",
+            "dest": "/etc/heka/heka-elasticsearch.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": "True"
+        },
         {
             "source": "{{ container_config_directory }}/heka-global.toml",
             "dest": "/etc/heka/heka-global.toml",
diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml
index c7c5cc986..947a1cb9f 100644
--- a/ansible/roles/elasticsearch/defaults/main.yml
+++ b/ansible/roles/elasticsearch/defaults/main.yml
@@ -2,8 +2,11 @@
 ####################
 # Elasticsearch
 ####################
-elasticsearch_port: "{{ elasticsearch_port }}"
-elasticsearch_host: "{{ kolla_internal_vip_address }}"
+elasticsearch_cluster_name: "kolla_logging"
+
+####################
+# Docker
+####################
 elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-elasticsearch"
 elasticsearch_tag: "{{ openstack_release }}"
 elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}"
diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
index eba662b63..449fb3c8a 100644
--- a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
+++ b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
@@ -1,4 +1,17 @@
-network.host: {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+{% set num_nodes = groups['elasticsearch'] | length %}
+{% set minimum_master_nodes = (num_nodes / 2 + 1) | round(0, 'floor') | int if num_nodes > 2 else 1 %}
+{% set recover_after_nodes = (num_nodes * 2 / 3) | round(0, 'floor') | int if num_nodes > 1 else 1 %}
+node.name: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+network.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+cluster.name: "{{ elasticsearch_cluster_name }}"
+node.master: true
+node.data: true
+discovery.zen.ping.unicast.hosts: [{% for host in groups['elasticsearch'] %}"{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"{% if not loop.last %},{% endif %}{% endfor %}]
+
+discovery.zen.minimum_master_nodes: {{ minimum_master_nodes }}
+gateway.expected_nodes: {{ num_nodes }}
+gateway.recover_after_time: "5m"
+gateway.recover_after_nodes: {{ recover_after_nodes }}
 path.conf: "/etc/elasticsearch"
 path.data: "/var/lib/elasticsearch/data"
 path.logs: "/var/log/elasticsearch"
diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2
index 1ab7dd67d..f3b76253d 100644
--- a/ansible/roles/haproxy/templates/haproxy.cfg.j2
+++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2
@@ -328,3 +328,26 @@ listen radosgw_external
 {% endfor %}
 {% endif %}
 {% endif %}
+
+{% if enable_central_logging | bool %}
+listen kibana
+  bind {{ kolla_internal_vip_address }}:{{ kibana_server_port }}
+{% for host in groups['kibana'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen kibana_external
+  bind {{ kolla_external_vip_address }}:{{ kibana_server_port }}
+{% for host in groups['kibana'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+
+listen elasticsearch
+  option dontlog-normal
+  bind {{ kolla_internal_vip_address }}:{{ elasticsearch_port }}
+{% for host in groups['elasticsearch'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
diff --git a/ansible/roles/kibana/defaults/main.yml b/ansible/roles/kibana/defaults/main.yml
index c720b0306..e35e2f1e7 100644
--- a/ansible/roles/kibana/defaults/main.yml
+++ b/ansible/roles/kibana/defaults/main.yml
@@ -2,12 +2,10 @@
 ####################
 # Kibana
 ####################
-kibana_port: "{{ kibana_port }}"
-kibana_host: "{{ kolla_internal_vip_address }}"
-kibana_app_id: "discover"
-kibana_request_timeout: 300000
-kibana_shard_timeout: 0
-kibana_verify_ssl: false
+kibana_default_app_id: "discover"
+kibana_elasticsearch_request_timeout: 300000
+kibana_elasticsearch_shard_timeout: 0
+kibana_elasticsearch_ssl_verify: false
 
 
 ####################
@@ -16,9 +14,3 @@ kibana_verify_ssl: false
 kibana_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-kibana"
 kibana_tag: "{{ openstack_release }}"
 kibana_image_full: "{{ kibana_image }}:{{ kibana_tag }}"
-
-
-####################
-# Elasticsearch
-####################
-elasticsearch_preserve_host: "true"
diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2
index 2fa66eec9..cc31234c9 100644
--- a/ansible/roles/kibana/templates/kibana.yml.j2
+++ b/ansible/roles/kibana/templates/kibana.yml.j2
@@ -1,11 +1,10 @@
-port: {{ kibana_port }}
-host: {{ kibana_host }}
-elasticsearch_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ elasticsearch_port }}"
-elasticsearch_preserve_host: {{ elasticsearch_preserve_host }}
-default_app_id: {{ kibana_app_id }}
-request_timeout: {{ kibana_request_timeout }}
-shard_timeout: {{ kibana_shard_timeout }}
-verify_ssl: {{ kibana_verify_ssl }}
+kibana.defaultAppId: "{{ kibana_default_app_id }}"
+server.port: {{ kibana_server_port }}
+server.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+elasticsearch.url: "{{ internal_protocol }}://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ elasticsearch_port }}"
+elasticsearch.requestTimeout: {{ kibana_elasticsearch_request_timeout }}
+elasticsearch.shardTimeout: {{ kibana_elasticsearch_shard_timeout }}
+elasticsearch.ssl.verify: {{ kibana_elasticsearch_ssl_verify }}
 bundled_plugin_ids:
  - plugins/dashboard/index
  - plugins/discover/index
diff --git a/ansible/site.yml b/ansible/site.yml
index ded4f6ef8..39d469c3f 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -30,13 +30,13 @@
   roles:
     - { role: kibana,
         tags: kibana,
-        when: enable_elk | bool }
+        when: enable_central_logging | bool }
 
 - hosts: elasticsearch
   roles:
     - { role: elasticsearch,
         tags: elasticsearch,
-        when: enable_elk | bool }
+        when: enable_central_logging | bool }
 
 - hosts: memcached
   roles:
diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua
index 440e9ab08..0296f6e23 100644
--- a/docker/heka/plugins/decoders/os_swift_log.lua
+++ b/docker/heka/plugins/decoders/os_swift_log.lua
@@ -20,7 +20,7 @@ local utils = require "os_utils"
 
 local msg = {
     Timestamp   = nil,
-    Type        = 'Syslog',
+    Type        = 'log',
     Hostname    = read_config("hostname"),
     Payload     = nil,
     Pid         = nil,
diff --git a/docker/heka/plugins/decoders/os_syslog.lua b/docker/heka/plugins/decoders/os_syslog.lua
index 34fdd0bcb..94e2744f8 100644
--- a/docker/heka/plugins/decoders/os_syslog.lua
+++ b/docker/heka/plugins/decoders/os_syslog.lua
@@ -20,7 +20,7 @@ local utils = require "os_utils"
 
 local msg = {
     Timestamp   = nil,
-    Type        = 'Syslog',
+    Type        = 'log',
     Hostname    = read_config("hostname"),
     Payload     = nil,
     Pid         = nil,
diff --git a/tools/cleanup-containers b/tools/cleanup-containers
index b5821d43e..a860c6780 100755
--- a/tools/cleanup-containers
+++ b/tools/cleanup-containers
@@ -33,7 +33,9 @@ else
         openvswitch_{vswitchd,db} \
         rabbitmq{,_bootstrap} \
         heka \
-        swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd}
+        swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd} \
+        elasticsearch \
+        kibana
     )
     ceph_osd_bootstrap=$(docker ps -a --filter "name=bootstrap_osd_*" --format "{{.Names}}")
     ceph_osd_containers=$(docker ps -a --filter "name=ceph_osd_*" --format "{{.Names}}")
@@ -51,7 +53,8 @@ else
         mongodb \
         haproxy_socket \
         heka{,_socket} \
-        kolla_logs
+        kolla_logs \
+        elasticsearch
     )
 fi
 
-- 
cgit v1.2.3