Use fluentd image labels

In order to orchestrate smooth transition to fluentd 0.14.x
aka 1.0 stable branch aka td-agent 3
from td-agent repository - use image labels (fluentd_version
and fluentd_binary).

Depends-On: https://review.opendev.org/676411
Change-Id: Iab8518c34ef876056c6abcdb5f2e9fc9f1f7dbdd

12 files changed, 68 insertions, 37 deletions

diff --git a/ansible/library/kolla_docker.py b/ansible/library/kolla_docker.py
index dee7a0df2..b9b1e724c 100644
--- a/ansible/library/kolla_docker.py
+++ b/ansible/library/kolla_docker.py
@@ -51,6 +51,7 @@ options:
       - compare_container
       - compare_image
       - create_volume
+      - ensure_image
       - get_container_env
       - get_container_state
       - pull_image
@@ -886,6 +887,10 @@ class DockerWorker(object):
                     )
                 raise
 
+    def ensure_image(self):
+        if not self.check_image():
+            self.pull_image()
+
 
 def generate_module():
     # NOTE(jeffrey4l): add empty string '' to choices let us use
@@ -893,13 +898,20 @@ def generate_module():
     argument_spec = dict(
         common_options=dict(required=False, type='dict', default=dict()),
         action=dict(required=True, type='str',
-                    choices=['compare_container', 'compare_image',
-                             'create_volume', 'get_container_env',
-                             'get_container_state', 'pull_image',
+                    choices=['compare_container',
+                             'compare_image',
+                             'create_volume',
+                             'ensure_image',
+                             'get_container_env',
+                             'get_container_state',
+                             'pull_image',
                              'recreate_or_restart_container',
-                             'remove_container', 'remove_image',
-                             'remove_volume', 'restart_container',
-                             'start_container', 'stop_container',
+                             'remove_container',
+                             'remove_image',
+                             'remove_volume',
+                             'restart_container',
+                             'start_container',
+                             'stop_container',
                              'stop_and_remove_container']),
         api_version=dict(required=False, type='str', default='auto'),
         auth_email=dict(required=False, type='str'),
@@ -947,6 +959,7 @@ def generate_module():
         ['action', 'compare_container', ['name']],
         ['action', 'compare_image', ['name']],
         ['action', 'create_volume', ['name']],
+        ['action', 'ensure_image', ['image']],
         ['action', 'get_container_env', ['name']],
         ['action', 'get_container_state', ['name']],
         ['action', 'recreate_or_restart_container', ['name']],
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 71fc70022..6ff7651ba 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -26,6 +26,32 @@
     - "fluentd/filter"
   when: enable_fluentd | bool
 
+- name: Ensure fluentd image is present for label check
+  vars:
+    service_name: "fluentd"
+    service: "{{ common_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "ensure_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ service.image }}"
+  when: enable_fluentd | bool
+
+- name: Fetch fluentd image labels
+  vars:
+    service_name: "fluentd"
+    service: "{{ common_services[service_name] }}"
+  become: true
+  docker_image_facts:
+    name: "{{ service.image }}"
+  register: fluentd_labels
+  when: enable_fluentd | bool
+
+- name: Set fluentd facts
+  set_fact:
+    fluentd_binary: "{% if fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary is not defined %}{% if kolla_base_distro in 'ubuntu' and ansible_architecture == 'x86_64' %}td-agent{% else %}fluentd{% endif %}{% else %}{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}{% endif %}"
+  when: enable_fluentd | bool
+
 - name: Copying over config.json files for services
   template:
     src: "{{ item.key }}.json.j2"
@@ -199,6 +225,7 @@
 - name: Copying over fluentd filter config files
   vars:
     customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
+    fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
   template:
     src: "conf/filter/{{ item.src }}.conf.j2"
     dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
@@ -207,7 +234,7 @@
   with_items:
     - src: 00-record_transformer
       dest: 00-record_transformer
-    - src: "{{ '01-rewrite-0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '01-rewrite-0.12' }}"
+    - src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
       dest: 01-rewrite
   when:
     - enable_fluentd | bool
diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2
index 36374d409..54b863f82 100644
--- a/ansible/roles/common/templates/conf/input/00-global.conf.j2
+++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2
@@ -1,5 +1,4 @@
 #jinja2: trim_blocks: False
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 {% set services = [
     ( 'aodh', enable_aodh ),
     ( 'barbican', enable_barbican ),
@@ -51,7 +50,7 @@
                 "/var/log/kolla/neutron/dnsmasq.log",
                 "/var/log/kolla/*/*-access.log",
                 "/var/log/kolla/*/*-error.log"]
-  pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
+  pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos
   tag kolla.*
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
index 86ff6fa99..3933a3856 100644
--- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
+++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/mariadb/mariadb.log
-  pos_file /var/run/{{ fluentd_dir }}/mariadb.pos
+  pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
   tag infra.mariadb
   format multiline
   format_firstline /^\d{6}/
diff --git a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
index 913f3755f..4cc414d9b 100644
--- a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
+++ b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/rabbitmq/rabbit.log
-  pos_file /var/run/{{ fluentd_dir }}/rabbit.pos
+  pos_file /var/run/{{ fluentd_binary }}/rabbit.pos
   tag infra.rabbit
   format multiline
   format_firstline /^\s*$/
diff --git a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
index 91b2ae33a..cf0fe8332 100644
--- a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
+++ b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
@@ -1,9 +1,8 @@
 # Note (blallau): to manage Apache and WSGI log files
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log
-  pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos
+  pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos
   tag kolla.*
   format /^(?<Payload>.*)$/
   enable_watch_timer false
diff --git a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
index ba4c40a43..040b94062 100644
--- a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
+++ b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/libvirt/libvirtd.log
-  pos_file /var/run/{{ fluentd_dir }}/libvirt.pos
+  pos_file /var/run/{{ fluentd_binary }}/libvirt.pos
   tag infra.libvirt
   format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
   time_key Timestamp
diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
index 49c733c63..3193227fd 100644
--- a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
+++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/zookeeper/zookeeper.log
-  pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos
+  pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos
   tag infra.*
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
diff --git a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2
index 22d22521a..ad086ba89 100644
--- a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2
+++ b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
-  pos_file /var/run/{{ fluentd_dir }}/kafka.pos
+  pos_file /var/run/{{ fluentd_binary }}/kafka.pos
   tag infra.*
   format multiline
   format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
diff --git a/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2 b/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
index 13c690682..43879c918 100644
--- a/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
+++ b/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/opendaylight/karaf.log
-  pos_file /var/run/{{ fluentd_dir }}/opendaylight.pos
+  pos_file /var/run/{{ fluentd_binary }}/opendaylight.pos
   tag infra.opendaylight
   format multiline
   format_firstline /\d{4}-\d{2}-\d{2}/
diff --git a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2
index 0d3c15383..7da68f4b9 100644
--- a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2
+++ b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
 <source>
   @type tail
   path /var/log/kolla/monasca/agent*.log
-  pos_file /var/run/{{ fluentd_dir }}/monasca-agent.pos
+  pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos
   tag kolla.*
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
@@ -13,7 +12,7 @@
 <source>
   @type tail
   path /var/log/kolla/monasca/grafana.log
-  pos_file /var/run/{{ fluentd_dir }}/monasca-grafana.pos
+  pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos
   tag infra.*
   format multiline
   format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 3ae1d5be0..40d2d14fb 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -1,18 +1,18 @@
-{% if kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64' %}
-    {% set fluentd_cmd = '/usr/sbin/td-agent' %}
-    {% set fluentd_dir = '/etc/td-agent' %}
-    {% set fluentd_conf = 'td-agent.conf' %}
-    {% set fluentd_user = 'td-agent' %}
-{% else %}
+{% set fluentd_user = fluentd_binary %}
+{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
+
+{% if fluentd_binary == 'fluentd' %}
+    {% set fluentd_conf = 'fluent.conf' %}
     {% if kolla_base_distro in ['ubuntu', 'debian'] %}
-        {% set fluentd_cmd = '/usr/local/bin/fluentd -c /etc/fluentd/fluent.conf' %}
+        {% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
     {% else %}
-        {% set fluentd_cmd = '/usr/bin/fluentd -c /etc/fluentd/fluent.conf' %}
+        {% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
     {% endif %}
-    {% set fluentd_dir = '/etc/fluentd' %}
-    {% set fluentd_conf = 'fluent.conf' %}
-    {% set fluentd_user = 'fluentd' %}
+{% elif fluentd_binary == 'td-agent' %}
+    {% set fluentd_conf = fluentd_binary ~ '.conf' %}
+    {% set fluentd_cmd = '/usr/sbin/td-agent' %}
 {% endif %}
+
 {
     "command": "{{ fluentd_cmd }}",
     "config_files": [