Merge "Clean fluentd_plugins_install block"

2 files changed, 47 insertions, 16 deletions

diff --git a/docker/fluentd/Dockerfile.j2 b/docker/fluentd/Dockerfile.j2
index 5b486b2f0..132508014 100644
--- a/docker/fluentd/Dockerfile.j2
+++ b/docker/fluentd/Dockerfile.j2
@@ -33,34 +33,54 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.install_packages(fluentd_packages | customizable("packages")) }}
 
-{% block fluentd_plugins_install %}
-
+# Distro specific files and operations
 {% if base_distro in  ['centos', 'oraclelinux', 'rhel'] %}
 COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-
-RUN ulimit -n 65536 \
-    && gem install --minimal-deps activesupport:4.2.9 public_suffix:2.0.5 fluent-plugin-parser fluent-plugin-kubernetes_metadata_filter fluent-plugin-elasticsearch:1.10.2 fluent-plugin-grep fluent-plugin-grok-parser:0.3.1 fluent-plugin-rewrite-tag-filter:1.6.0 fluent-plugin-secure-forward \
-    && chmod -R 440 /etc/sudoers.d/kolla_fluentd_sudoers  \
-    && chmod 755 /usr/local/bin/kolla_extend_start \
+RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
     && mkdir -p /var/run/fluentd \
     && chown -R fluentd: /etc/fluentd /var/run/fluentd
-
 {% elif base_distro in ['debian', 'ubuntu'] %}
-RUN ulimit -n 65536 \
+COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
+RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
     && sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \
-    && td-agent-gem install fluent-plugin-parser fluent-plugin-kubernetes_metadata_filter fluent-plugin-elasticsearch fluent-plugin-grep fluent-plugin-grok-parser:2.1.4 fluent-plugin-rewrite-tag-filter:2.0.0 fluent-plugin-secure-forward \
-    && rm -f /etc/td-agent/td-agent.conf
+    && rm -f /etc/td-agent/td-agent.conf \
+    && chown -R td-agent: /etc/td-agent
+{% endif %}
+
 
-COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start
 
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
-    && chmod 755 /usr/local/bin/kolla_extend_start \
-    && chown -R td-agent: /etc/td-agent
+{% block fluentd_plugins_install %}
 
+
+{% set fluentd_plugins = [
+    'fluent-plugin-parser',
+    'fluent-plugin-kubernetes_metadata_filter',
+    'fluent-plugin-grep',
+    'fluent-plugin-secure-forward'
+] %}
+
+{% if base_distro in  ['centos', 'oraclelinux', 'rhel'] %}
+    # NOTE: We use fluentd version is v12.0 so fluent-plugin-grok-parse version should be < 1.0.0.
+    # https://github.com/fluent/fluent-plugin-grok-parser
+    {% set fluentd_plugins = [
+        'activesupport:4.2.9',
+        'public_suffix:2.0.5',
+        'fluent-plugin-elasticsearch:1.10.2',
+        'fluent-plugin-grok-parser:0.3.1',
+        'fluent-plugin-rewrite-tag-filter:1.6.0'
+    ] + fluentd_plugins  %}
+{% elif base_distro in ['debian', 'ubuntu'] %}
+    {% set fluentd_plugins = [
+        'fluent-plugin-elasticsearch',
+        'fluent-plugin-grok-parser:2.1.4',
+        'fluent-plugin-rewrite-tag-filter:2.0.0'
+    ] + fluentd_plugins %}
 {% endif %}
 
+{{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
+
 {% endblock %}
 
 {% block fluentd_footer %}{% endblock %}
diff --git a/docker/macros.j2 b/docker/macros.j2
index 5294ddacf..dfb85447b 100644
--- a/docker/macros.j2
+++ b/docker/macros.j2
@@ -80,3 +80,14 @@ RUN apt-get update \
     RUN curl -o /usr/bin/kubectl http://storage.googleapis.com/kubernetes-release/release/v1.5.4/bin/linux/${KUBE_ARCH}/kubectl \
         && chmod 755 /usr/bin/kubectl
 {% endmacro %}
+
+
+{% macro install_fluent_plugins(plugins) -%}
+{% if plugins is defined and plugins|length > 0 -%}
+    {% if base_distro in ['centos', 'oraclelinux', 'rhel'] -%}
+        RUN ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
+    {%- elif base_distro in ['debian', 'ubuntu'] -%}
+        RUN ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
+    {%- endif %}
+{%- endif %}
+{%- endmacro %}