Ceph Ansible support

Add the initial playbooks for making ceph and ansible play nice
together.

This does not include all of the openstack changes to make things like
nova, glance, and cinder work. This will simply build the ceph cluster
and thats it. The next patchset will do the OpenStack integration.

DocImpact
Change-Id: Ie1697dde5f92e833652933a80f0004f31b641330
Partially-Implements: blueprint ceph-container

7 files changed, 149 insertions, 17 deletions

diff --git a/docker/ceph/ceph-base/Dockerfile.j2 b/docker/ceph/ceph-base/Dockerfile.j2
index b18890b5d..363346bce 100644
--- a/docker/ceph/ceph-base/Dockerfile.j2
+++ b/docker/ceph/ceph-base/Dockerfile.j2
@@ -15,3 +15,7 @@ RUN apt-get install -y --no-install-recommends \
     && apt-get clean
 
 {% endif %}
+
+RUN useradd --user-group ceph \
+    && mkdir -p /home/ceph \
+    && chown -R ceph: /home/ceph
diff --git a/docker/ceph/ceph-mon/Dockerfile.j2 b/docker/ceph/ceph-mon/Dockerfile.j2
index e9d47fc70..e8a1337ac 100644
--- a/docker/ceph/ceph-mon/Dockerfile.j2
+++ b/docker/ceph/ceph-mon/Dockerfile.j2
@@ -3,5 +3,6 @@ MAINTAINER Kolla Project (https://launchpad.net/kolla)
 
 COPY start.sh /
 COPY config-external.sh /opt/kolla/
+COPY fetch_ceph_keys.py /usr/bin/
 
 CMD ["/start.sh"]
diff --git a/docker/ceph/ceph-mon/config-external.sh b/docker/ceph/ceph-mon/config-external.sh
index 75488239c..289cffc86 100644
--- a/docker/ceph/ceph-mon/config-external.sh
+++ b/docker/ceph/ceph-mon/config-external.sh
@@ -8,3 +8,33 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.mon.keyring"
+TARGET="/etc/ceph/ceph.client.mon.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.monmap"
+TARGET="/etc/ceph/ceph.monmap"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-mon/fetch_ceph_keys.py b/docker/ceph/ceph-mon/fetch_ceph_keys.py
new file mode 100755
index 000000000..ab2be9c16
--- /dev/null
+++ b/docker/ceph/ceph-mon/fetch_ceph_keys.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a stripped down version of an ansible module I wrote in Yaodu to
+# achieve the same goals we have for Kolla. I have relicensed it for Kolla
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp
+
+# Basically this module will fetch the admin and mon keyrings as well as the
+# monmap file. It then hashes the content, compresses them, and finally it
+# converts them to base64 to be safely transported around with ansible
+
+import base64
+import hashlib
+import json
+import os
+import sys
+import zlib
+
+
+def json_exit(msg=None, failed=False, changed=False):
+    if type(msg) is not dict:
+        msg = {'msg': str(msg)}
+    msg.update({'failed': failed, 'changed': changed})
+    print(json.dumps(msg))
+    sys.exit()
+
+
+def read_file(filename):
+    filename_path = os.path.join('/etc/ceph', filename)
+
+    if not os.path.exists(filename_path):
+        json_exit("file not found: {}".format(filename_path), failed=True)
+    if not os.access(filename_path, os.R_OK):
+        json_exit("file not readable: {}".format(filename_path), failed=True)
+
+    with open(filename_path, 'rb') as f:
+        raw_data = f.read()
+
+    return {'content': base64.b64encode(zlib.compress(raw_data)),
+            'sha1': hashlib.sha1(raw_data).hexdigest(),
+            'filename': filename}
+
+
+def main():
+    admin_keyring = 'ceph.client.admin.keyring'
+    mon_keyring = 'ceph.client.mon.keyring'
+    monmap = 'ceph.monmap'
+
+    files = [admin_keyring, mon_keyring, monmap]
+    json_exit({filename: read_file(filename) for filename in files})
+
+
+if __name__ == '__main__':
+    main()
diff --git a/docker/ceph/ceph-mon/start.sh b/docker/ceph/ceph-mon/start.sh
index eb5d7f5af..3afaa3ad4 100644..100755
--- a/docker/ceph/ceph-mon/start.sh
+++ b/docker/ceph/ceph-mon/start.sh
@@ -3,11 +3,11 @@
 set -o errexit
 
 CMD="/usr/bin/ceph-mon"
-ARGS="-d -i ${MON_NAME} --public-addr ${MON_IP}:6789"
+ARGS="-d -i $(hostname) --public-addr ${MON_IP}:6789"
 
 # Setup common paths
-KEYRING_ADMIN="/etc/ceph/ceph.admin.keyring"
-KEYRING_MON="/etc/ceph/ceph.mon.keyring"
+KEYRING_ADMIN="/etc/ceph/ceph.client.admin.keyring"
+KEYRING_MON="/etc/ceph/ceph.client.mon.keyring"
 MONMAP="/etc/ceph/ceph.monmap"
 MON_DIR="/var/lib/ceph/mon/ceph-$(hostname)"
 
@@ -21,7 +21,7 @@ set_configs
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     # Lookup our fsid from the ceph.conf
-    FSID="$(awk '/^fsid/ {print $3; exit}' ${ceph_conf})"
+    FSID="$(awk '/^fsid/ {print $3; exit}' /etc/ceph/ceph.conf)"
 
     # Generating initial keyrings and monmap
     ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *'
@@ -29,8 +29,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}"
     monmaptool --create --add "$(hostname)" "${MON_IP}" --fsid "${FSID}" "${MONMAP}"
 
-    # TODO(SamYaple): Return json parsible output to ansible
-    exit 0
+    echo "Sleeping until keys are fetched"
+    /bin/sleep infinity
 fi
 
 # This section runs on every mon that does not have a keyring already.
diff --git a/docker/ceph/ceph-osd/config-external.sh b/docker/ceph/ceph-osd/config-external.sh
index 8e17487ed..8928f91cb 100644
--- a/docker/ceph/ceph-osd/config-external.sh
+++ b/docker/ceph/ceph-osd/config-external.sh
@@ -8,3 +8,13 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-osd/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-osd/start.sh b/docker/ceph/ceph-osd/start.sh
index cf9114a62..84549ae8c 100644..100755
--- a/docker/ceph/ceph-osd/start.sh
+++ b/docker/ceph/ceph-osd/start.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
+set -o xtrace
 set -o errexit
 
-CMD="/usr/bin/ceph-osd"
-ARGS="-f -d -i ${OSD_ID} --osd-journal ${OSD_DIR}/journal -k ${OSD_DIR}/keyring"
-
 # Loading common functions.
 source /opt/kolla/kolla-common.sh
 
@@ -13,23 +11,45 @@ set_configs
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    # Creating a new label for the disk
-    parted "${OSD_DEV}" -s -- mklabel gpt
+    JOURNAL_PARTUUID=$(uuidgen)
+
+    # Formating disk for ceph
+    sgdisk --zap-all -- "${OSD_DEV}"
+    sgdisk --new=2:1M:5G --change-name=2:KOLLA_CEPH_JOURNAL --typecode=2:45B0969E-9B03-4F30-B4C6-B4B80CEFF106 --partition-guid=2:${JOURNAL_PARTUUID} --mbrtogpt -- "${OSD_DEV}"
+    sgdisk --largest-new=1 --change-name=1:KOLLA_CEPH_DATA --typecode=1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -- "${OSD_DEV}"
+    # This command may throw errors that we can safely ignore
+    partprobe || true
+
+    # We look up the appropriate device path with partition.
+    OSD_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?1)"
+    JOURNAL_PARTITION="${OSD_PARTITION%?}2"
 
-    # Preparing the OSD for use with Ceph
-    ceph-disk prepare "${OSD_DEV}"
     OSD_ID="$(ceph osd create)"
     OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
     mkdir -p "${OSD_DIR}"
-    mount "${OSD_DEV}1" "${OSD_DIR}"
-    ceph-osd -i "${OSD_ID}" --mkfs --mkkey
-    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow proflie osd' -i "${OSD_DIR}/keyring"
+    mkfs.xfs -f "${OSD_PARTITION}"
+    mount "${OSD_PARTITION}" "${OSD_DIR}"
 
-    # Adding osd to crush map
+    # This will through an error about no key existing. That is normal. It then
+    # creates the key in the next step.
+    ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey
+    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
+    umount "${OSD_PARTITION}"
+
+    # These commands only need to be run once per host but are safe to run
+    # repeatedly. This can be improved later or if any problems arise.
     ceph osd crush add-bucket "$(hostname)" host
     ceph osd crush move "$(hostname)" root=default
+
+    # Adding osd to crush map
     ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="$(hostname)"
     exit 0
 fi
 
+# We look up the appropriate journal since we cannot rely on symlinks
+JOURNAL_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?2)"
+OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
+CMD="/usr/bin/ceph-osd"
+ARGS="-f -d -i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
+
 exec $CMD $ARGS