Merge "Make Heka send logs to Elasticsearch"

15 files changed, 98 insertions, 34 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index ff0e4d2f0..30f4b3172 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -130,7 +130,7 @@ rgw_port: "6780"
 
 mistral_api_port: "8989"
 
-kibana_port: "5601"
+kibana_server_port: "5601"
 
 elasticsearch_port: "9200"
 
@@ -184,7 +184,7 @@ enable_murano: "no"
 enable_ironic: "no"
 enable_magnum: "no"
 enable_mistral: "no"
-enable_elk: "no"
+enable_central_logging: "no"
 enable_mongodb: "no"
 enable_manila: "no"
 
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 18f87c610..8592801ab 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -46,3 +46,11 @@
     - "swift-object-updater"
     - "swift-proxy-server"
     - "swift-rsyncd"
+
+- name: Copying over heka elasticsearch config file
+  template:
+    src: "heka-{{ item }}.toml.j2"
+    dest: "{{ node_config_directory }}/heka/heka-{{ item }}.toml"
+  with_items:
+    - "elasticsearch"
+  when: "{{ enable_central_logging | bool }}"
diff --git a/ansible/roles/common/templates/heka-elasticsearch.toml.j2 b/ansible/roles/common/templates/heka-elasticsearch.toml.j2
new file mode 100644
index 000000000..7b9a351cb
--- /dev/null
+++ b/ansible/roles/common/templates/heka-elasticsearch.toml.j2
@@ -0,0 +1,16 @@
+[elasticsearch_json_encoder]
+type = "ESJsonEncoder"
+index = {{'"%{Type}-%{%Y.%m.%d}"'}}
+es_index_from_timestamp = true
+fields = ["Timestamp", "Type", "Logger", "Severity", "Payload", "Pid", "Hostname", "DynamicFields"]
+
+[elasticsearch_output]
+type = "ElasticSearchOutput"
+server = "{{ internal_protocol }}://{{ kolla_internal_vip_address }}:{{ elasticsearch_port }}"
+message_matcher = "Type == 'log'"
+encoder = "elasticsearch_json_encoder"
+use_buffering = true
+    [elasticsearch_output.buffering]
+    max_buffer_size = 1073741824  # 1024 * 1024 * 1024
+    max_file_size = 134217728  # 128 * 1024 * 1024
+    full_action = "drop"
diff --git a/ansible/roles/common/templates/heka-haproxy.toml.j2 b/ansible/roles/common/templates/heka-haproxy.toml.j2
index d933a6f55..f30dd32d5 100644
--- a/ansible/roles/common/templates/heka-haproxy.toml.j2
+++ b/ansible/roles/common/templates/heka-haproxy.toml.j2
@@ -3,12 +3,12 @@
 
 [haproxy_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:haproxy)/"
+message_matcher = "Fields[programname] =~ /(?i:haproxy)/"
 path = "/var/log/kolla/haproxy/haproxy.log"
 encoder = "syslog_encoder"
 
 [keepalived_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:keepalived)/"
+message_matcher = "Fields[programname] =~ /(?i:keepalived)/"
 path = "/var/log/kolla/haproxy/keepalived.log"
 encoder = "syslog_encoder"
diff --git a/ansible/roles/common/templates/heka-swift.toml.j2 b/ansible/roles/common/templates/heka-swift.toml.j2
index 3bb520e32..8ed4a1e23 100644
--- a/ansible/roles/common/templates/heka-swift.toml.j2
+++ b/ansible/roles/common/templates/heka-swift.toml.j2
@@ -3,6 +3,6 @@
 
 [{{ item }}_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] == '{{ item }}'"
+message_matcher = "Fields[programname] == '{{ item }}'"
 path = "/var/log/kolla/swift/{{ item }}.log"
 encoder = "syslog_encoder"
diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2
index f5bcad297..332790231 100644
--- a/ansible/roles/common/templates/heka.json.j2
+++ b/ansible/roles/common/templates/heka.json.j2
@@ -3,6 +3,13 @@
     "command": "/usr/bin/hekad -config=/etc/heka/",
     "config_files": [
         {
+            "source": "{{ container_config_directory }}/heka-elasticsearch.toml",
+            "dest": "/etc/heka/heka-elasticsearch.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": "True"
+        },
+        {
             "source": "{{ container_config_directory }}/heka-global.toml",
             "dest": "/etc/heka/heka-global.toml",
             "owner": "heka",
diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml
index c7c5cc986..947a1cb9f 100644
--- a/ansible/roles/elasticsearch/defaults/main.yml
+++ b/ansible/roles/elasticsearch/defaults/main.yml
@@ -2,8 +2,11 @@
 ####################
 # Elasticsearch
 ####################
-elasticsearch_port: "{{ elasticsearch_port }}"
-elasticsearch_host: "{{ kolla_internal_vip_address }}"
+elasticsearch_cluster_name: "kolla_logging"
+
+####################
+# Docker
+####################
 elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-elasticsearch"
 elasticsearch_tag: "{{ openstack_release }}"
 elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}"
diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
index eba662b63..449fb3c8a 100644
--- a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
+++ b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
@@ -1,4 +1,17 @@
-network.host: {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+{% set num_nodes = groups['elasticsearch'] | length %}
+{% set minimum_master_nodes = (num_nodes / 2 + 1) | round(0, 'floor') | int if num_nodes > 2 else 1 %}
+{% set recover_after_nodes = (num_nodes * 2 / 3) | round(0, 'floor') | int if num_nodes > 1 else 1 %}
+node.name: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+network.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+cluster.name: "{{ elasticsearch_cluster_name }}"
+node.master: true
+node.data: true
+discovery.zen.ping.unicast.hosts: [{% for host in groups['elasticsearch'] %}"{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"{% if not loop.last %},{% endif %}{% endfor %}]
+
+discovery.zen.minimum_master_nodes: {{ minimum_master_nodes }}
+gateway.expected_nodes: {{ num_nodes }}
+gateway.recover_after_time: "5m"
+gateway.recover_after_nodes: {{ recover_after_nodes }}
 path.conf: "/etc/elasticsearch"
 path.data: "/var/lib/elasticsearch/data"
 path.logs: "/var/log/elasticsearch"
diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2
index a9f1e8ac6..56d34e79b 100644
--- a/ansible/roles/haproxy/templates/haproxy.cfg.j2
+++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2
@@ -376,3 +376,26 @@ listen radosgw_external
 {% endfor %}
 {% endif %}
 {% endif %}
+
+{% if enable_central_logging | bool %}
+listen kibana
+  bind {{ kolla_internal_vip_address }}:{{ kibana_server_port }}
+{% for host in groups['kibana'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen kibana_external
+  bind {{ kolla_external_vip_address }}:{{ kibana_server_port }}
+{% for host in groups['kibana'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+
+listen elasticsearch
+  option dontlog-normal
+  bind {{ kolla_internal_vip_address }}:{{ elasticsearch_port }}
+{% for host in groups['elasticsearch'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
diff --git a/ansible/roles/kibana/defaults/main.yml b/ansible/roles/kibana/defaults/main.yml
index c720b0306..e35e2f1e7 100644
--- a/ansible/roles/kibana/defaults/main.yml
+++ b/ansible/roles/kibana/defaults/main.yml
@@ -2,12 +2,10 @@
 ####################
 # Kibana
 ####################
-kibana_port: "{{ kibana_port }}"
-kibana_host: "{{ kolla_internal_vip_address }}"
-kibana_app_id: "discover"
-kibana_request_timeout: 300000
-kibana_shard_timeout: 0
-kibana_verify_ssl: false
+kibana_default_app_id: "discover"
+kibana_elasticsearch_request_timeout: 300000
+kibana_elasticsearch_shard_timeout: 0
+kibana_elasticsearch_ssl_verify: false
 
 
 ####################
@@ -16,9 +14,3 @@ kibana_verify_ssl: false
 kibana_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-kibana"
 kibana_tag: "{{ openstack_release }}"
 kibana_image_full: "{{ kibana_image }}:{{ kibana_tag }}"
-
-
-####################
-# Elasticsearch
-####################
-elasticsearch_preserve_host: "true"
diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2
index 2fa66eec9..cc31234c9 100644
--- a/ansible/roles/kibana/templates/kibana.yml.j2
+++ b/ansible/roles/kibana/templates/kibana.yml.j2
@@ -1,11 +1,10 @@
-port: {{ kibana_port }}
-host: {{ kibana_host }}
-elasticsearch_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ elasticsearch_port }}"
-elasticsearch_preserve_host: {{ elasticsearch_preserve_host }}
-default_app_id: {{ kibana_app_id }}
-request_timeout: {{ kibana_request_timeout }}
-shard_timeout: {{ kibana_shard_timeout }}
-verify_ssl: {{ kibana_verify_ssl }}
+kibana.defaultAppId: "{{ kibana_default_app_id }}"
+server.port: {{ kibana_server_port }}
+server.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+elasticsearch.url: "{{ internal_protocol }}://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ elasticsearch_port }}"
+elasticsearch.requestTimeout: {{ kibana_elasticsearch_request_timeout }}
+elasticsearch.shardTimeout: {{ kibana_elasticsearch_shard_timeout }}
+elasticsearch.ssl.verify: {{ kibana_elasticsearch_ssl_verify }}
 bundled_plugin_ids:
  - plugins/dashboard/index
  - plugins/discover/index
diff --git a/ansible/site.yml b/ansible/site.yml
index cf1141239..f84e40b9d 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -30,13 +30,13 @@
   roles:
     - { role: kibana,
         tags: kibana,
-        when: enable_elk | bool }
+        when: enable_central_logging | bool }
 
 - hosts: elasticsearch
   roles:
     - { role: elasticsearch,
         tags: elasticsearch,
-        when: enable_elk | bool }
+        when: enable_central_logging | bool }
 
 - hosts: memcached
   roles:
diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua
index 440e9ab08..0296f6e23 100644
--- a/docker/heka/plugins/decoders/os_swift_log.lua
+++ b/docker/heka/plugins/decoders/os_swift_log.lua
@@ -20,7 +20,7 @@ local utils = require "os_utils"
 
 local msg = {
     Timestamp   = nil,
-    Type        = 'Syslog',
+    Type        = 'log',
     Hostname    = read_config("hostname"),
     Payload     = nil,
     Pid         = nil,
diff --git a/docker/heka/plugins/decoders/os_syslog.lua b/docker/heka/plugins/decoders/os_syslog.lua
index 34fdd0bcb..94e2744f8 100644
--- a/docker/heka/plugins/decoders/os_syslog.lua
+++ b/docker/heka/plugins/decoders/os_syslog.lua
@@ -20,7 +20,7 @@ local utils = require "os_utils"
 
 local msg = {
     Timestamp   = nil,
-    Type        = 'Syslog',
+    Type        = 'log',
     Hostname    = read_config("hostname"),
     Payload     = nil,
     Pid         = nil,
diff --git a/tools/cleanup-containers b/tools/cleanup-containers
index b5821d43e..a860c6780 100755
--- a/tools/cleanup-containers
+++ b/tools/cleanup-containers
@@ -33,7 +33,9 @@ else
         openvswitch_{vswitchd,db} \
         rabbitmq{,_bootstrap} \
         heka \
-        swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd}
+        swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd} \
+        elasticsearch \
+        kibana
     )
     ceph_osd_bootstrap=$(docker ps -a --filter "name=bootstrap_osd_*" --format "{{.Names}}")
     ceph_osd_containers=$(docker ps -a --filter "name=ceph_osd_*" --format "{{.Names}}")
@@ -51,7 +53,8 @@ else
         mongodb \
         haproxy_socket \
         heka{,_socket} \
-        kolla_logs
+        kolla_logs \
+        elasticsearch
     )
 fi