Merge "Fix Swift logging"

7 files changed, 66 insertions, 6 deletions

diff --git a/ansible/roles/common/templates/heka-global.toml.j2 b/ansible/roles/common/templates/heka-global.toml.j2
index c9fbc1e7e..5e325026a 100644
--- a/ansible/roles/common/templates/heka-global.toml.j2
+++ b/ansible/roles/common/templates/heka-global.toml.j2
@@ -11,9 +11,20 @@ filename = "lua_decoders/os_syslog.lua"
   [syslog_log_decoder.config]
   hostname = "{{ ansible_hostname }}"
 
+[swift_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_swift_log.lua"
+  [swift_log_decoder.config]
+  hostname = "{{ ansible_hostname }}"
+
+[multi_log_decoder]
+type = "MultiDecoder"
+subs = ["syslog_log_decoder", "swift_log_decoder"]
+cascade_strategy = "first-wins"
+
 [syslog_input]
 type = "UdpInput"
 net = "unixgram"
 address = "/var/lib/kolla/heka/log"
-decoder = "syslog_log_decoder"
+decoder = "multi_log_decoder"
 splitter = "NullSplitter"
diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2
index 30ed99880..34471008d 100644
--- a/ansible/roles/common/templates/heka.json.j2
+++ b/ansible/roles/common/templates/heka.json.j2
@@ -46,7 +46,7 @@
             "perm": "0600"
         }
 {%- if not loop.last %},{% endif %}
-{% endfor %}
+{% endfor -%}
 
     ]
 }
diff --git a/ansible/roles/swift/templates/account.conf.j2 b/ansible/roles/swift/templates/account.conf.j2
index d3c2f2644..ad54582cc 100644
--- a/ansible/roles/swift/templates/account.conf.j2
+++ b/ansible/roles/swift/templates/account.conf.j2
@@ -6,7 +6,7 @@ mount_check = false
 log_address = /var/lib/kolla/heka/log
 log_name = {{ service_name }}
 log_facility = LOG_LOCAL0
-log_level = warning
+log_level = INFO
 
 [pipeline:main]
 pipeline = account-server
diff --git a/ansible/roles/swift/templates/container.conf.j2 b/ansible/roles/swift/templates/container.conf.j2
index 879dd9c3c..55e0976dd 100644
--- a/ansible/roles/swift/templates/container.conf.j2
+++ b/ansible/roles/swift/templates/container.conf.j2
@@ -6,7 +6,7 @@ mount_check = false
 log_address = /var/lib/kolla/heka/log
 log_name = {{ service_name }}
 log_facility = LOG_LOCAL0
-log_level = warning
+log_level = INFO
 
 [pipeline:main]
 pipeline = container-server
diff --git a/ansible/roles/swift/templates/object.conf.j2 b/ansible/roles/swift/templates/object.conf.j2
index b3856228c..7499bbfd2 100644
--- a/ansible/roles/swift/templates/object.conf.j2
+++ b/ansible/roles/swift/templates/object.conf.j2
@@ -7,7 +7,7 @@ mount_check = false
 log_address = /var/lib/kolla/heka/log
 log_name = {{ service_name }}
 log_facility = LOG_LOCAL0
-log_level = warning
+log_level = INFO
 
 [pipeline:main]
 {% if service_name == 'swift-object-expirer' %}
diff --git a/ansible/roles/swift/templates/proxy-server.conf.j2 b/ansible/roles/swift/templates/proxy-server.conf.j2
index e8ea502e2..cf9de196f 100644
--- a/ansible/roles/swift/templates/proxy-server.conf.j2
+++ b/ansible/roles/swift/templates/proxy-server.conf.j2
@@ -5,7 +5,7 @@ bind_port = {{ swift_proxy_server_port }}
 log_address = /var/lib/kolla/heka/log
 log_name = {{ service_name }}
 log_facility = LOG_LOCAL0
-log_level = warning
+log_level = INFO
 
 [pipeline:main]
 pipeline = catch_errors gatekeeper healthcheck cache container_sync bulk ratelimit authtoken keystoneauth slo dlo proxy-server
diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua
new file mode 100644
index 000000000..440e9ab08
--- /dev/null
+++ b/docker/heka/plugins/decoders/os_swift_log.lua
@@ -0,0 +1,49 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- The code in this file was inspired by Heka's rsyslog.lua decoder plugin.
+-- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua
+
+local syslog = require "syslog"
+local utils = require "os_utils"
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'Syslog',
+    Hostname    = read_config("hostname"),
+    Payload     = nil,
+    Pid         = nil,
+    Severity    = nil,
+    Fields      = nil
+}
+
+-- See https://github.com/openstack/swift/blob/2a8b455/swift/common/utils.py#L1423-L1424
+local swift_grammar = syslog.build_rsyslog_grammar('<%PRI%>%programname%: %msg%')
+
+function process_message ()
+    local log = read_message("Payload")
+
+    local fields = swift_grammar:match(log)
+    if not fields then return -1 end
+
+    msg.Severity = fields.pri.severity
+    fields.syslogfacility = fields.pri.facility
+    fields.pri = nil
+
+    msg.Payload = fields.msg
+    fields.msg = nil
+
+    msg.Fields = fields
+    return utils.safe_inject_message(msg)
+end