Merge "Ceph Ansible support"

26 files changed, 761 insertions, 19 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 3a4419723..e4e903f9f 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -140,6 +140,7 @@ enable_nova: "yes"
 enable_rabbitmq: "yes"
 
 # Additional optional OpenStack services are specified here
+enable_ceph: "no"
 enable_cinder: "no"
 enable_heat: "yes"
 enable_horizon: "yes"
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 8b782f72a..d820c3e60 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -54,6 +54,12 @@ control
 [murano:children]
 control
 
+[ceph-mon:children]
+control
+
+[ceph-osd:children]
+storage
+
 
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 924c4ac84..51f9d055c 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -62,6 +62,12 @@ control
 [murano:children]
 control
 
+[ceph-mon:children]
+control
+
+[ceph-osd:children]
+storage
+
 
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
diff --git a/ansible/library/bslurp.py b/ansible/library/bslurp.py
new file mode 100644
index 000000000..485b4c3b4
--- /dev/null
+++ b/ansible/library/bslurp.py
@@ -0,0 +1,189 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module has been relicensed from the source below:
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp
+
+DOCUMENTATION = '''
+---
+module: bslurp
+short_description: Slurps a file from a remote node
+description:
+     - Used for fetching a binary blob containing the file, then push that file
+       to other hosts.
+options:
+  src:
+    description:
+      - File to fetch. When dest is used, src is expected to be a str with data
+    required: True
+    type: str
+  compress:
+    description:
+      - Compress file with zlib
+    default: True
+    type: bool
+  dest:
+    description:
+      - Where to write out binary blob
+    required: False
+    type: str
+  mode:
+    description:
+      - Destination file permissions
+    default: '0644'
+    type: str
+  sha1:
+    description:
+      - sha1 hash of the underlying data
+    default: None
+    type: bool
+author: Sam Yaple
+'''
+
+EXAMPLES = '''
+Distribute a file from single to many host:
+
+- hosts: web_servers
+  tasks:
+    - name: Pull in web config
+      bslurp: src="/path/to/file"
+      register: file_data
+      run_once: True
+    - name: Push if changed
+      bslurp:
+        src: "{{ file_data.content }}"
+        dest: "{{ file_data.source }}"
+        mode: "{{ file_data.mode }}"
+        sha1: "{{ file_data.sha1 }}"
+
+Distribute multiple files from single to many host:
+
+- hosts: web_servers
+  tasks:
+    - name: Pull in web config
+      bslurp: src="{{ item }}"
+      with_items:
+        - "/path/to/file1"
+        - "/path/to/file2"
+        - "/path/to/file3"
+      register: file_data
+      run_once: True
+    - name: Push if changed
+      bslurp:
+        src: "{{ item.content }}"
+        dest: "{{ item.source }}"
+        mode: "{{ item.mode }}"
+        sha1: "{{ item.sha1 }}"
+      with_items: file_data.results
+
+Distribute a to file many host without compression; Change permissions on dest:
+
+- hosts: web_servers
+  tasks:
+    - name: Pull in web config
+      bslurp: src="/path/to/file"
+      register: file_data
+      run_once: True
+    - name: Push if changed
+      bslurp:
+        src: "{{ file_data.content }}"
+        dest: "/new/path/to/file"
+        mode: "0777"
+        compress: False
+        sha1: "{{ file_data.sha1 }}"
+'''
+
+import base64
+import hashlib
+import os
+import sys
+import zlib
+
+def copy_from_host(module):
+    compress = module.params.get('compress')
+    src = module.params.get('src')
+
+    if not os.path.exists(src):
+        module.fail_json(msg="file not found: {}".format(src))
+    if not os.access(src, os.R_OK):
+        module.fail_json(msg="file is not readable: {}".format(src))
+
+    mode = oct(os.stat(src).st_mode & 0777)
+
+    with open(src, 'rb') as f:
+        raw_data = f.read()
+
+    sha1 = hashlib.sha1(raw_data).hexdigest()
+    data = zlib.compress(raw_data) if compress else raw_data
+
+    module.exit_json(content=base64.b64encode(data), sha1=sha1, mode=mode,
+                     source=src)
+
+
+def copy_to_host(module):
+    compress = module.params.get('compress')
+    dest = module.params.get('dest')
+    mode = int(module.params.get('mode'), 0)
+    sha1 = module.params.get('sha1')
+    src = module.params.get('src')
+
+    data = base64.b64decode(src)
+    raw_data = zlib.decompress(data) if compress else data
+
+    if sha1:
+        if os.path.exists(dest):
+            if os.access(dest, os.R_OK):
+                with open(dest, 'rb') as f:
+                    if hashlib.sha1(f.read()).hexdigest() == sha1:
+                        module.exit_json(changed=False)
+            else:
+                module.exit_json(failed=True, changed=False,
+                                 msg='file is not accessible: {}'.format(dest))
+
+        if sha1 != hashlib.sha1(raw_data).hexdigest():
+            module.exit_json(failed=True, changed=False,
+                             msg='sha1 sum does not match data')
+
+    with os.fdopen(os.open(dest, os.O_WRONLY | os.O_CREAT, mode), 'wb') as f:
+        f.write(raw_data)
+
+    module.exit_json(changed=True)
+
+def main():
+    module = AnsibleModule(
+        argument_spec = dict(
+            compress = dict(default=True, type='bool'),
+            dest = dict(type='str'),
+            mode = dict(default='0644', type='str'),
+            sha1 = dict(default=None, type='str'),
+            src = dict(required=True, type='str')
+        )
+    )
+
+    dest = module.params.get('dest')
+
+    try:
+        if dest:
+            copy_to_host(module)
+        else:
+            copy_from_host(module)
+    except Exception as e:
+        module.exit_json(failed=True, changed=True, msg=repr(e))
+
+# import module snippets
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+    main()
diff --git a/ansible/library/find_disks.py b/ansible/library/find_disks.py
new file mode 100644
index 000000000..fe2a638b2
--- /dev/null
+++ b/ansible/library/find_disks.py
@@ -0,0 +1,86 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module has been relicensed from the source below:
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/ceph_osd_list
+
+DOCUMENTATION = '''
+---
+module: find_disks
+short_description: Return list of devices containing a specfied label
+description:
+     - This will return a list of all devices with a GPT partition label with
+       the name specified.
+options:
+  partition_name:
+    description:
+      - Partition name
+    required: True
+    type: bool
+author: Sam Yaple
+'''
+
+EXAMPLES = '''
+- hosts: ceph-osd
+  tasks:
+    - name: Return all valid formated devices with the name KOLLA_CEPH_OSD
+      ceph_osd_list:
+          partition_name: 'KOLLA_CEPH_OSD'
+      register: osds
+'''
+
+import sys
+import subprocess
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec = dict(
+            partition_name = dict(required=True, type='str')
+        )
+    )
+
+    partition_name = module.params.get('partition_name')
+
+    try:
+        # This should all really be done differently. Unfortunately there is no
+        # decent python library for dealing with disks like we need to here.
+        disks = subprocess.check_output("parted -l", shell=True).split('\n')
+        ret = list()
+
+        for line in disks:
+            d = line.split(' ')
+            if d[0] == 'Disk':
+                dev = d[1][:-1]
+
+            if line.find(partition_name) != -1:
+                # This process returns an error code when no results return
+                # We can ignore that, it is safe
+                p = subprocess.Popen("blkid " + dev + "*", shell=True, stdout=subprocess.PIPE)
+                fs_uuid = p.communicate()[0]
+                # The dev doesn't need to have a uuid, will be '' otherwise
+                if fs_uuid:
+                    fs_uuid = fs_uuid.split('"')[1]
+                ret.append({'device': dev, 'fs_uuid': fs_uuid})
+
+        module.exit_json(disks=ret)
+    except Exception as e:
+        module.exit_json(failed=True, msg=repr(e))
+
+# import module snippets
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+    main()
diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml
new file mode 100644
index 000000000..08fffde13
--- /dev/null
+++ b/ansible/roles/ceph/defaults/main.yml
@@ -0,0 +1,24 @@
+---
+project_name: "ceph"
+
+
+####################
+# Docker
+####################
+ceph_mon_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceph-mon"
+ceph_mon_tag: "{{ openstack_release }}"
+ceph_mon_image_full: "{{ ceph_mon_image }}:{{ ceph_mon_tag }}"
+
+ceph_osd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceph-osd"
+ceph_osd_tag: "{{ openstack_release }}"
+ceph_osd_image_full: "{{ ceph_osd_image }}:{{ ceph_osd_tag }}"
+
+ceph_data_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-data"
+ceph_data_tag: "{{ openstack_release }}"
+ceph_data_image_full: "{{ ceph_data_image }}:{{ ceph_data_tag }}"
+
+
+####################
+# Ceph
+####################
+osd_initial_weight: "1"
diff --git a/ansible/roles/ceph/tasks/bootstrap_mons.yml b/ansible/roles/ceph/tasks/bootstrap_mons.yml
new file mode 100644
index 000000000..853056b88
--- /dev/null
+++ b/ansible/roles/ceph/tasks/bootstrap_mons.yml
@@ -0,0 +1,57 @@
+---
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_ceph_cluster state=absent
+  changed_when: False
+  always_run: True
+  run_once: True
+
+- name: Creating temp file on localhost
+  local_action: copy content=None dest=/tmp/kolla_ceph_cluster mode=0600
+  changed_when: False
+  always_run: True
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when check
+- name: Checking if a previous cluster exists
+  command: docker exec ceph_mon_data stat /etc/ceph/ceph.monmap
+  register: exists
+  changed_when: False
+  failed_when: False
+  always_run: True
+
+- name: Writing hostname of host with existing cluster files to temp file
+  local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_ceph_cluster mode=0600
+  changed_when: False
+  always_run: True
+  when: exists.rc == 0
+
+- name: Registering host from temp file
+  set_fact:
+    delegate_host: "{{ lookup('file', '/tmp/kolla_ceph_cluster') }}"
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_ceph_cluster state=absent
+  changed_when: False
+  always_run: True
+  run_once: True
+
+- name: Starting Ceph Monitor data container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: ceph_mon_data
+    image: "{{ ceph_data_image_full }}"
+    volumes:
+      - "/etc/ceph/"
+      - "/var/lib/ceph/"
+
+- include: generate_cluster.yml
+  when: delegate_host == 'None' and inventory_hostname == groups['ceph-mon'][0]
diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml
new file mode 100644
index 000000000..59163f8d3
--- /dev/null
+++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml
@@ -0,0 +1,48 @@
+---
+- name: Looking up disks to bootstrap for Ceph
+  find_disks:
+      partition_name: 'KOLLA_CEPH_OSD_BOOTSTRAP'
+  register: osds_bootstrap
+  when: inventory_hostname in groups['ceph-osd']
+
+- name: Bootstrapping Ceph OSDs
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "no"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    privileged: True
+    name: "bootstrap_osd_{{ item.0 }}"
+    image: "{{ ceph_osd_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/ceph-osd/:/opt/kolla/ceph-osd/:ro"
+      - "/dev/:/dev/"
+    env:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      OSD_DEV: "{{ item.1.device }}"
+      OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
+  with_indexed_items: osds_bootstrap['disks']|default([])
+  when: inventory_hostname in groups['ceph-osd']
+
+# https://github.com/ansible/ansible-modules-core/pull/1031
+- name: Waiting for bootstrap containers to exit
+  command: docker wait "bootstrap_osd_{{ item.0 }}"
+  register: bootstrap_result
+  run_once: True
+  failed_when: bootstrap_result.stdout != "0"
+  with_indexed_items: osds_bootstrap['disks']|default([])
+  when: inventory_hostname in groups['ceph-osd']
+
+- name: Cleaning up bootstrap containers
+  docker:
+    name: "bootstrap_osd_{{ item.0 }}"
+    image: "{{ ceph_osd_image_full }}"
+    state: absent
+  with_indexed_items: osds_bootstrap['disks']|default([])
+  when: inventory_hostname in groups['ceph-osd']
diff --git a/ansible/roles/ceph/tasks/config.yml b/ansible/roles/ceph/tasks/config.yml
new file mode 100644
index 000000000..d242ea80c
--- /dev/null
+++ b/ansible/roles/ceph/tasks/config.yml
@@ -0,0 +1,22 @@
+---
+- include: ../../config.yml
+  vars:
+    service_name: "ceph-mon"
+    config_source:
+      - "roles/ceph/templates/ceph.conf.j2"
+      - "/etc/kolla/config/ceph.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/ceph.conf"
+
+- include: ../../config.yml
+  vars:
+    service_name: "ceph-osd"
+    config_source:
+      - "roles/ceph/templates/ceph.conf.j2"
+      - "/etc/kolla/config/ceph.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/ceph.conf"
diff --git a/ansible/roles/ceph/tasks/distribute_keyrings.yml b/ansible/roles/ceph/tasks/distribute_keyrings.yml
new file mode 100644
index 000000000..10ec0a8dd
--- /dev/null
+++ b/ansible/roles/ceph/tasks/distribute_keyrings.yml
@@ -0,0 +1,34 @@
+---
+- name: Fetching Ceph keyrings
+  command: docker exec ceph_mon fetch_ceph_keys.py
+  register: ceph_files_json
+  changed_when: "{{ (ceph_files_json.stdout | from_json).changed }}"
+  failed_when: "{{ (ceph_files_json.stdout | from_json).failed }}"
+  delegate_to: "{{ delegate_host }}"
+  run_once: True
+
+- name: Reading json from variable
+  set_fact:
+    ceph_files: "{{ (ceph_files_json.stdout | from_json) }}"
+
+- name: Pushing Ceph keyring for OSDs
+  bslurp:
+    src: "{{ item.content }}"
+    dest: "{{ node_config_directory }}/ceph-osd/{{ item.filename }}"
+    mode: 0600
+    sha1: "{{ item.sha1 }}"
+  with_items:
+    - "{{ ceph_files['ceph.client.admin.keyring'] }}"
+  when: inventory_hostname in groups['ceph-osd']
+
+- name: Pushing Ceph keyrings for Mons
+  bslurp:
+    src: "{{ item.content }}"
+    dest: "{{ node_config_directory }}/ceph-mon/{{ item.filename }}"
+    mode: 0600
+    sha1: "{{ item.sha1 }}"
+  with_items:
+    - "{{ ceph_files['ceph.client.admin.keyring'] }}"
+    - "{{ ceph_files['ceph.client.mon.keyring'] }}"
+    - "{{ ceph_files['ceph.monmap'] }}"
+  when: inventory_hostname in groups['ceph-mon']
diff --git a/ansible/roles/ceph/tasks/generate_cluster.yml b/ansible/roles/ceph/tasks/generate_cluster.yml
new file mode 100644
index 000000000..1a0f16b1b
--- /dev/null
+++ b/ansible/roles/ceph/tasks/generate_cluster.yml
@@ -0,0 +1,30 @@
+---
+- name: Generating Initial Ceph keyrings and monmap
+  docker:
+    detach: False
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "no"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: ceph_mon
+    image: "{{ ceph_mon_image_full }}"
+    volumes: "{{ node_config_directory }}/ceph-mon/:/opt/kolla/ceph-mon/:ro"
+    volumes_from:
+      - "ceph_mon_data"
+    env:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+
+- name: Waiting for a few settings for cluster to generate keys
+  command: sleep 3
+  changed_when: False
+
+- name: Setting host for cluster files
+  set_fact:
+    delegate_host: "{{ ansible_hostname }}"
diff --git a/ansible/roles/ceph/tasks/main.yml b/ansible/roles/ceph/tasks/main.yml
new file mode 100644
index 000000000..63dca3460
--- /dev/null
+++ b/ansible/roles/ceph/tasks/main.yml
@@ -0,0 +1,16 @@
+---
+- include: config.yml
+
+- include: bootstrap_mons.yml
+  when: inventory_hostname in groups['ceph-mon']
+
+- include: distribute_keyrings.yml
+
+- include: start_mons.yml
+  when: inventory_hostname in groups['ceph-mon']
+
+- include: bootstrap_osds.yml
+  when: inventory_hostname in groups['ceph-osd']
+
+- include: start_osds.yml
+  when: inventory_hostname in groups['ceph-osd']
diff --git a/ansible/roles/ceph/tasks/start_mons.yml b/ansible/roles/ceph/tasks/start_mons.yml
new file mode 100644
index 000000000..ab50d7355
--- /dev/null
+++ b/ansible/roles/ceph/tasks/start_mons.yml
@@ -0,0 +1,22 @@
+---
+- name: Starting ceph-mon container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: ceph_mon
+    image: "{{ ceph_mon_image_full }}"
+    volumes: "{{ node_config_directory }}/ceph-mon/:/opt/kolla/ceph-mon/:ro"
+    volumes_from:
+      - "ceph_mon_data"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+  when: inventory_hostname in groups['ceph-mon']
diff --git a/ansible/roles/ceph/tasks/start_osds.yml b/ansible/roles/ceph/tasks/start_osds.yml
new file mode 100644
index 000000000..593ec7b15
--- /dev/null
+++ b/ansible/roles/ceph/tasks/start_osds.yml
@@ -0,0 +1,50 @@
+---
+- name: Looking up OSDs for Ceph
+  find_disks:
+      partition_name: 'KOLLA_CEPH_DATA'
+  register: osds
+
+- name: Mounting Ceph OSD volumes
+  mount:
+    src: "UUID={{ item.fs_uuid }}"
+    fstype: xfs
+    state: mounted
+    name: "/var/lib/ceph/osd/{{ item.fs_uuid }}"
+  with_items: osds.disks
+
+- name: Gathering OSD IDs
+  command: 'cat /var/lib/ceph/osd/{{ item.fs_uuid }}/whoami'
+  with_items: osds.disks
+  register: id
+  changed_when: False
+  failed_when: id.rc != 0
+
+- name: Starting ceph-osds container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    pid: host
+    privileged: True
+    name: "ceph_osd_{{ item.0.stdout }}"
+    image: "{{ ceph_osd_image_full }}"
+    volumes:
+      - "/var/lib/ceph/osd/{{ item.1.fs_uuid }}:/var/lib/ceph/osd/ceph-{{ item.0.stdout }}"
+      - "{{ node_config_directory }}/ceph-osd/:/opt/kolla/ceph-osd/:ro"
+      - "/dev/:/dev/"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      OSD_ID: "{{ item.0.stdout }}"
+      OSD_DEV: "{{ item.1.device }}"
+  with_together:
+    - id.results
+    - osds.disks
+  when: inventory_hostname in groups['ceph-osd']
+        and osds.disks
diff --git a/ansible/roles/ceph/templates/ceph.conf.j2 b/ansible/roles/ceph/templates/ceph.conf.j2
new file mode 100644
index 000000000..9518742d7
--- /dev/null
+++ b/ansible/roles/ceph/templates/ceph.conf.j2
@@ -0,0 +1,9 @@
+[global]
+fsid = {{ ceph_cluster_fsid }}
+mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+
+mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+
+auth cluster required = cephx
+auth service required = cephx
+auth client required = cephx
diff --git a/ansible/site.yml b/ansible/site.yml
index 5052000b6..3ba043b8e 100755
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -1,4 +1,8 @@
 ---
+- hosts: [ceph-mon, ceph-osd]
+  roles:
+    - { role: ceph, tags: ceph, when: enable_ceph | bool }
+
 - hosts: [haproxy, mariadb, rabbitmq, cinder-api, glance-api, keystone, nova-api, neutron-server, swift-proxy-server]
   roles:
     - { role: haproxy, tags: haproxy, when: enable_haproxy | bool }
diff --git a/docker/ceph/ceph-base/Dockerfile.j2 b/docker/ceph/ceph-base/Dockerfile.j2
index b18890b5d..363346bce 100644
--- a/docker/ceph/ceph-base/Dockerfile.j2
+++ b/docker/ceph/ceph-base/Dockerfile.j2
@@ -15,3 +15,7 @@ RUN apt-get install -y --no-install-recommends \
     && apt-get clean
 
 {% endif %}
+
+RUN useradd --user-group ceph \
+    && mkdir -p /home/ceph \
+    && chown -R ceph: /home/ceph
diff --git a/docker/ceph/ceph-mon/Dockerfile.j2 b/docker/ceph/ceph-mon/Dockerfile.j2
index e9d47fc70..e8a1337ac 100644
--- a/docker/ceph/ceph-mon/Dockerfile.j2
+++ b/docker/ceph/ceph-mon/Dockerfile.j2
@@ -3,5 +3,6 @@ MAINTAINER Kolla Project (https://launchpad.net/kolla)
 
 COPY start.sh /
 COPY config-external.sh /opt/kolla/
+COPY fetch_ceph_keys.py /usr/bin/
 
 CMD ["/start.sh"]
diff --git a/docker/ceph/ceph-mon/config-external.sh b/docker/ceph/ceph-mon/config-external.sh
index 75488239c..289cffc86 100644
--- a/docker/ceph/ceph-mon/config-external.sh
+++ b/docker/ceph/ceph-mon/config-external.sh
@@ -8,3 +8,33 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.mon.keyring"
+TARGET="/etc/ceph/ceph.client.mon.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.monmap"
+TARGET="/etc/ceph/ceph.monmap"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-mon/fetch_ceph_keys.py b/docker/ceph/ceph-mon/fetch_ceph_keys.py
new file mode 100755
index 000000000..ab2be9c16
--- /dev/null
+++ b/docker/ceph/ceph-mon/fetch_ceph_keys.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a stripped down version of an ansible module I wrote in Yaodu to
+# achieve the same goals we have for Kolla. I have relicensed it for Kolla
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp
+
+# Basically this module will fetch the admin and mon keyrings as well as the
+# monmap file. It then hashes the content, compresses them, and finally it
+# converts them to base64 to be safely transported around with ansible
+
+import base64
+import hashlib
+import json
+import os
+import sys
+import zlib
+
+
+def json_exit(msg=None, failed=False, changed=False):
+    if type(msg) is not dict:
+        msg = {'msg': str(msg)}
+    msg.update({'failed': failed, 'changed': changed})
+    print(json.dumps(msg))
+    sys.exit()
+
+
+def read_file(filename):
+    filename_path = os.path.join('/etc/ceph', filename)
+
+    if not os.path.exists(filename_path):
+        json_exit("file not found: {}".format(filename_path), failed=True)
+    if not os.access(filename_path, os.R_OK):
+        json_exit("file not readable: {}".format(filename_path), failed=True)
+
+    with open(filename_path, 'rb') as f:
+        raw_data = f.read()
+
+    return {'content': base64.b64encode(zlib.compress(raw_data)),
+            'sha1': hashlib.sha1(raw_data).hexdigest(),
+            'filename': filename}
+
+
+def main():
+    admin_keyring = 'ceph.client.admin.keyring'
+    mon_keyring = 'ceph.client.mon.keyring'
+    monmap = 'ceph.monmap'
+
+    files = [admin_keyring, mon_keyring, monmap]
+    json_exit({filename: read_file(filename) for filename in files})
+
+
+if __name__ == '__main__':
+    main()
diff --git a/docker/ceph/ceph-mon/start.sh b/docker/ceph/ceph-mon/start.sh
index eb5d7f5af..3afaa3ad4 100644..100755
--- a/docker/ceph/ceph-mon/start.sh
+++ b/docker/ceph/ceph-mon/start.sh
@@ -3,11 +3,11 @@
 set -o errexit
 
 CMD="/usr/bin/ceph-mon"
-ARGS="-d -i ${MON_NAME} --public-addr ${MON_IP}:6789"
+ARGS="-d -i $(hostname) --public-addr ${MON_IP}:6789"
 
 # Setup common paths
-KEYRING_ADMIN="/etc/ceph/ceph.admin.keyring"
-KEYRING_MON="/etc/ceph/ceph.mon.keyring"
+KEYRING_ADMIN="/etc/ceph/ceph.client.admin.keyring"
+KEYRING_MON="/etc/ceph/ceph.client.mon.keyring"
 MONMAP="/etc/ceph/ceph.monmap"
 MON_DIR="/var/lib/ceph/mon/ceph-$(hostname)"
 
@@ -21,7 +21,7 @@ set_configs
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     # Lookup our fsid from the ceph.conf
-    FSID="$(awk '/^fsid/ {print $3; exit}' ${ceph_conf})"
+    FSID="$(awk '/^fsid/ {print $3; exit}' /etc/ceph/ceph.conf)"
 
     # Generating initial keyrings and monmap
     ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *'
@@ -29,8 +29,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}"
     monmaptool --create --add "$(hostname)" "${MON_IP}" --fsid "${FSID}" "${MONMAP}"
 
-    # TODO(SamYaple): Return json parsible output to ansible
-    exit 0
+    echo "Sleeping until keys are fetched"
+    /bin/sleep infinity
 fi
 
 # This section runs on every mon that does not have a keyring already.
diff --git a/docker/ceph/ceph-osd/config-external.sh b/docker/ceph/ceph-osd/config-external.sh
index 8e17487ed..8928f91cb 100644
--- a/docker/ceph/ceph-osd/config-external.sh
+++ b/docker/ceph/ceph-osd/config-external.sh
@@ -8,3 +8,13 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-osd/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-osd/start.sh b/docker/ceph/ceph-osd/start.sh
index cf9114a62..84549ae8c 100644..100755
--- a/docker/ceph/ceph-osd/start.sh
+++ b/docker/ceph/ceph-osd/start.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
+set -o xtrace
 set -o errexit
 
-CMD="/usr/bin/ceph-osd"
-ARGS="-f -d -i ${OSD_ID} --osd-journal ${OSD_DIR}/journal -k ${OSD_DIR}/keyring"
-
 # Loading common functions.
 source /opt/kolla/kolla-common.sh
 
@@ -13,23 +11,45 @@ set_configs
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    # Creating a new label for the disk
-    parted "${OSD_DEV}" -s -- mklabel gpt
+    JOURNAL_PARTUUID=$(uuidgen)
+
+    # Formating disk for ceph
+    sgdisk --zap-all -- "${OSD_DEV}"
+    sgdisk --new=2:1M:5G --change-name=2:KOLLA_CEPH_JOURNAL --typecode=2:45B0969E-9B03-4F30-B4C6-B4B80CEFF106 --partition-guid=2:${JOURNAL_PARTUUID} --mbrtogpt -- "${OSD_DEV}"
+    sgdisk --largest-new=1 --change-name=1:KOLLA_CEPH_DATA --typecode=1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -- "${OSD_DEV}"
+    # This command may throw errors that we can safely ignore
+    partprobe || true
+
+    # We look up the appropriate device path with partition.
+    OSD_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?1)"
+    JOURNAL_PARTITION="${OSD_PARTITION%?}2"
 
-    # Preparing the OSD for use with Ceph
-    ceph-disk prepare "${OSD_DEV}"
     OSD_ID="$(ceph osd create)"
     OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
     mkdir -p "${OSD_DIR}"
-    mount "${OSD_DEV}1" "${OSD_DIR}"
-    ceph-osd -i "${OSD_ID}" --mkfs --mkkey
-    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow proflie osd' -i "${OSD_DIR}/keyring"
+    mkfs.xfs -f "${OSD_PARTITION}"
+    mount "${OSD_PARTITION}" "${OSD_DIR}"
 
-    # Adding osd to crush map
+    # This will through an error about no key existing. That is normal. It then
+    # creates the key in the next step.
+    ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey
+    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
+    umount "${OSD_PARTITION}"
+
+    # These commands only need to be run once per host but are safe to run
+    # repeatedly. This can be improved later or if any problems arise.
     ceph osd crush add-bucket "$(hostname)" host
     ceph osd crush move "$(hostname)" root=default
+
+    # Adding osd to crush map
     ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="$(hostname)"
     exit 0
 fi
 
+# We look up the appropriate journal since we cannot rely on symlinks
+JOURNAL_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?2)"
+OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
+CMD="/usr/bin/ceph-osd"
+ARGS="-f -d -i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
+
 exec $CMD $ARGS
diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2
index 1880512af..f7dec9e21 100644
--- a/docker/openstack-base/Dockerfile.j2
+++ b/docker/openstack-base/Dockerfile.j2
@@ -40,7 +40,6 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
         python-tuskarclient \
         python-zaqarclient \
         python-openstackclient \
-        MySQL-python \
-        numpy
+        MySQL-python
 
 {% endif %}
diff --git a/etc/kolla/config/ceph.conf b/etc/kolla/config/ceph.conf
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/etc/kolla/config/ceph.conf
diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml
index 3d7ec924f..c2c077b1c 100644
--- a/etc/kolla/passwords.yml
+++ b/etc/kolla/passwords.yml
@@ -4,6 +4,12 @@
 
 
 ###################
+# Ceph options
+####################
+ceph_cluster_fsid: "5fba2fbc-551d-11e5-a8ce-01ef4c5cf93c"
+
+
+###################
 # Database options
 ####################
 database_password: "password"
@@ -46,6 +52,7 @@ heat_domain_admin_password: "password"
 murano_database_password: "password"
 murano_keystone_password: "password"
 
+
 ####################
 # RabbitMQ options
 ####################