Merge "Added Elasticearch and its deployment."

19 files changed, 146 insertions, 1 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 6cce5c447..d18a1285a 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -126,6 +126,7 @@ rgw_port: "6780"
 mistral_api_port: "8989"
 
 kibana_port: "5601"
+
 elasticsearch_port: "9200"
 
 public_protocol: "http"
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 5e8a2187e..a44eb1d66 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -18,6 +18,9 @@ localhost       ansible_connection=local
 [kibana:children]
 control
 
+[elasticsearch:children]
+control
+
 [haproxy:children]
 network
 
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 0265867db..3491b1ec9 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -30,6 +30,9 @@ storage01
 [kibana:children]
 control
 
+[elasticsearch:children]
+control
+
 [haproxy:children]
 network
 
diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml
new file mode 100644
index 000000000..032573671
--- /dev/null
+++ b/ansible/roles/elasticsearch/defaults/main.yml
@@ -0,0 +1,9 @@
+---
+####################
+# Elasticsearch
+####################
+elasticsearch_port: "{{ elasticsearch_port }}"
+elasticsearch_host: "{{ kolla_internal_address }}"
+elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-elasticsearch"
+elasticsearch_tag: "{{ openstack_release }}"
+elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}"
diff --git a/ansible/roles/elasticsearch/meta/main.yml b/ansible/roles/elasticsearch/meta/main.yml
new file mode 100644
index 000000000..6b4fff8fe
--- /dev/null
+++ b/ansible/roles/elasticsearch/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }
diff --git a/ansible/roles/elasticsearch/tasks/config.yml b/ansible/roles/elasticsearch/tasks/config.yml
new file mode 100644
index 000000000..97b4dc5f0
--- /dev/null
+++ b/ansible/roles/elasticsearch/tasks/config.yml
@@ -0,0 +1,22 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "elasticsearch"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+  with_items:
+    - "elasticsearch"
+
+- name: Copying over elasticsearch.yml
+  template:
+    src: "elasticsearch.yml.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/{{ item }}.yml"
+  with_items:
+    - "elasticsearch"
diff --git a/ansible/roles/elasticsearch/tasks/deploy.yml b/ansible/roles/elasticsearch/tasks/deploy.yml
new file mode 100644
index 000000000..1f16915ad
--- /dev/null
+++ b/ansible/roles/elasticsearch/tasks/deploy.yml
@@ -0,0 +1,4 @@
+---
+- include: config.yml
+
+- include: start.yml
diff --git a/ansible/roles/elasticsearch/tasks/main.yml b/ansible/roles/elasticsearch/tasks/main.yml
new file mode 100644
index 000000000..b017e8b4a
--- /dev/null
+++ b/ansible/roles/elasticsearch/tasks/main.yml
@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"
diff --git a/ansible/roles/elasticsearch/tasks/pull.yml b/ansible/roles/elasticsearch/tasks/pull.yml
new file mode 100644
index 000000000..6dae5004a
--- /dev/null
+++ b/ansible/roles/elasticsearch/tasks/pull.yml
@@ -0,0 +1,6 @@
+---
+- name: Pulling elasticsearch image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ elasticsearch_image_full }}"
diff --git a/ansible/roles/elasticsearch/tasks/start.yml b/ansible/roles/elasticsearch/tasks/start.yml
new file mode 100644
index 000000000..a0ac0ca51
--- /dev/null
+++ b/ansible/roles/elasticsearch/tasks/start.yml
@@ -0,0 +1,10 @@
+---
+- name: Starting Elasticsearch container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ elasticsearch_image_full }}"
+    name: "elasticsearch"
+    volumes:
+      - "{{ node_config_directory }}/elasticsearch/:{{ container_config_directory }}/"
+      - "elasticsearch:/var/lib/elasticsearch/data"
diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.json.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.json.j2
new file mode 100644
index 000000000..d87adeb3f
--- /dev/null
+++ b/ansible/roles/elasticsearch/templates/elasticsearch.json.j2
@@ -0,0 +1,11 @@
+{
+    "command": "/usr/share/elasticsearch/bin/elasticsearch",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/elasticsearch.yml",
+            "dest": "/usr/share/elasticsearch/config/elasticsearch.yml",
+            "owner": "elasticsearch",
+            "perm": "0600"
+        }
+    ]
+}
diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
new file mode 100644
index 000000000..eba662b63
--- /dev/null
+++ b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
@@ -0,0 +1,5 @@
+network.host: {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+path.conf: "/etc/elasticsearch"
+path.data: "/var/lib/elasticsearch/data"
+path.logs: "/var/log/elasticsearch"
+path.scripts: "/etc/elasticsearch/scripts"
diff --git a/ansible/site.yml b/ansible/site.yml
index 4bb48e0b4..bebc8b666 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -32,6 +32,12 @@
         tags: kibana,
         when: enable_elk | bool }
 
+- hosts: elasticsearch
+  roles:
+    - { role: elasticsearch,
+        tags: elasticsearch,
+        when: enable_elk | bool }
+
 - hosts: memcached
   roles:
     - { role: memcached,
diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index 4fe02e7e1..2b394b400 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -18,6 +18,8 @@ ENV KOLLA_INSTALL_METATYPE {{ install_metatype }}
 {% if base_distro in ['fedora', 'centos', 'oraclelinux', 'rhel'] %}
 
 #### BEGIN REPO ENABLEMENT
+# Turns on Elasticsearch repos
+COPY elasticsearch.yum.repo /etc/yum.repos.d/elasticsearch.repo
 
 # Turns on MariaDB repos throughout the RPM build
 COPY mariadb.yum.repo /etc/yum.repos.d/MariaDB.repo
@@ -28,7 +30,8 @@ RUN yum -y install http://repo.percona.com/release/7/RPMS/x86_64/percona-release
 
 RUN rpm --import http://yum.mariadb.org/RPM-GPG-KEY-MariaDB \
     && rpm --import https://download.ceph.com/keys/release.asc \
-    && rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-Percona
+    && rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-Percona \
+    && rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
 
     {% if install_metatype in ['rdo', 'mixed'] %}
 
@@ -153,6 +156,7 @@ RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 199369E540
     && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 391A9AA2147192839E9DB0315EDB1B62EC4926EA \
     && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A \
     && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 08B73419AC32B4E966C1A330E84AC2C0460F3994 \
+    && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 46095ACC8548582C1A2699A9D27D666CD88E42B4 \
     && apt-get update \
     && apt-get upgrade -y \
     && apt-get dist-upgrade -y \
diff --git a/docker/base/elasticsearch.yum.repo b/docker/base/elasticsearch.yum.repo
new file mode 100644
index 000000000..e9d28a615
--- /dev/null
+++ b/docker/base/elasticsearch.yum.repo
@@ -0,0 +1,6 @@
+[elasticsearch-2.x]
+name=Elasticsearch repository for 2.x packages
+baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
+gpgcheck=1
+gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
+enabled=1
diff --git a/docker/base/sources.list b/docker/base/sources.list
index d63541f02..3a6684490 100644
--- a/docker/base/sources.list
+++ b/docker/base/sources.list
@@ -18,3 +18,6 @@ deb http://repo.percona.com/apt trusty main
 
 # Ceph repo
 deb http://download.ceph.com/debian-hammer/ trusty main
+
+# Elastic repo
+deb http://packages.elastic.co/elasticsearch/2.x/debian stable main
diff --git a/docker/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/Dockerfile.j2
new file mode 100644
index 000000000..aa9f43c1b
--- /dev/null
+++ b/docker/elasticsearch/Dockerfile.j2
@@ -0,0 +1,40 @@
+FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
+MAINTAINER {{ maintainer }}
+
+
+{% if base_distro in ['centos', 'fedora', 'oraclelinux', 'rhel'] %}
+
+ENV JAVA_HOME /usr/lib/jvm/jre-1.7.0-openjdk/
+
+RUN yum -y install \
+        java-1.7.0-openjdk \
+        elasticsearch-2.1.1 \
+    && yum clean all
+
+{% elif base_distro in ['ubuntu', 'debian'] %}
+
+ENV JAVA_HOME /usr/lib/jvm/java-1.7.0-openjdk-amd64/
+
+RUN apt-get install -y --no-install-recommends \
+       openjdk-7-jre \
+       elasticsearch=2.1.1 \
+    && apt-get clean
+
+{% endif %}
+
+COPY elasticsearch_sudoers /etc/sudoers.d/elasticsearch_sudoers
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start \
+    && chmod 750 /etc/sudoers.d \
+    && chmod 440 /etc/sudoers.d/elasticsearch_sudoers \
+    && usermod -a -G kolla elasticsearch
+
+
+# by default elasticsearch shell is /bin/false, we need
+# /bin/bash to run elasticsearch as non-root
+# https://discuss.elastic.co/t/running-as-non-root-user-service-wrapper-has-changed/7863
+RUN usermod -s /bin/bash elasticsearch -d /usr/share/elasticsearch
+
+{{ include_footer }}
+
+USER elasticsearch
diff --git a/docker/elasticsearch/elasticsearch_sudoers b/docker/elasticsearch/elasticsearch_sudoers
new file mode 100644
index 000000000..76396c68a
--- /dev/null
+++ b/docker/elasticsearch/elasticsearch_sudoers
@@ -0,0 +1 @@
+%kolla ALL=(root) NOPASSWD: /bin/chown elasticsearch\: /var/lib/elasticsearch/data, /usr/bin/chown elasticsearch\: /var/lib/elasticsearch/data
diff --git a/docker/elasticsearch/extend_start.sh b/docker/elasticsearch/extend_start.sh
new file mode 100644
index 000000000..ac78966fb
--- /dev/null
+++ b/docker/elasticsearch/extend_start.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Only update permissions if permissions need to be updated
+if [[ $(stat -c %U:%G /var/lib/elasticsearch/data) != "elasticsearch:elasticsearch" ]]; then
+    sudo chown elasticsearch: /var/lib/elasticsearch/data
+fi