From 56a07702bcea02cf3793d6c830acc687b68244be Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 2 Jul 2020 18:17:10 +0100 Subject: Performance: use a single config file for fluentd Currently we generate multiple fluentd configuration files for inputs, filters, formatters and outputs. These are then included from the main td-agent.conf configuration file. With a large number of hosts, this can take a long time to template. Benchmarking of templating is available at [1]. This change switches to a single fluentd configuration file, with the include done locally. For the default template files included with Kolla Ansible we use Jinja includes, but this does not work with templates in a different directory. We therefore use the Ansible template lookup plugin, which has a slightly higher overhead than a jinja include, but far lower than generating multiple templates. This should drastically improve the performance of this task. [1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md Partially-Implements: blueprint performance-improvements Change-Id: Ia8623be0aa861fea3e54d2c9e1c971dfd8e3afa9 --- kolla_ansible/tests/unit/test_fluentd_filters.py | 96 ++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 kolla_ansible/tests/unit/test_fluentd_filters.py (limited to 'kolla_ansible/tests') diff --git a/kolla_ansible/tests/unit/test_fluentd_filters.py b/kolla_ansible/tests/unit/test_fluentd_filters.py new file mode 100644 index 000000000..353d01732 --- /dev/null +++ b/kolla_ansible/tests/unit/test_fluentd_filters.py @@ -0,0 +1,96 @@ +# Copyright (c) 2020 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from kolla_ansible.fluentd_filters import customise_fluentd + + +class TestFilters(unittest.TestCase): + + def test_customise_fluentd_no_files(self): + default_files = [ + ] + customised_files = [ + ] + expected = [ + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_no_customised_files(self): + default_files = [ + "foo/bar.conf.j2" + ] + customised_files = [ + ] + expected = [ + "foo/bar.conf.j2" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_no_default_files(self): + default_files = [ + ] + customised_files = [ + "foo/bar.conf" + ] + expected = [ + "foo/bar.conf" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_both(self): + default_files = [ + "foo/bar.conf.j2" + ] + customised_files = [ + "baz/qux.conf" + ] + expected = [ + "foo/bar.conf.j2", + "baz/qux.conf" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_override(self): + default_files = [ + "foo/bar.conf.j2" + ] + customised_files = [ + "baz/bar.conf" + ] + expected = [ + "baz/bar.conf" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_both_with_override(self): + default_files = [ + "foo/bar.conf.j2", + "baz/qux.conf.j2" + ] + customised_files = [ + "baz/bar.conf" + ] + expected = [ + "baz/bar.conf", + "baz/qux.conf.j2" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) -- cgit v1.2.3