Merge "Ceph Ansible support"

7 files changed, 149 insertions, 17 deletions

diff --git a/docker/ceph/ceph-base/Dockerfile.j2 b/docker/ceph/ceph-base/Dockerfile.j2
index b18890b5d..363346bce 100644
--- a/docker/ceph/ceph-base/Dockerfile.j2
+++ b/docker/ceph/ceph-base/Dockerfile.j2
@@ -15,3 +15,7 @@ RUN apt-get install -y --no-install-recommends \
     && apt-get clean
 
 {% endif %}
+
+RUN useradd --user-group ceph \
+    && mkdir -p /home/ceph \
+    && chown -R ceph: /home/ceph
diff --git a/docker/ceph/ceph-mon/Dockerfile.j2 b/docker/ceph/ceph-mon/Dockerfile.j2
index e9d47fc70..e8a1337ac 100644
--- a/docker/ceph/ceph-mon/Dockerfile.j2
+++ b/docker/ceph/ceph-mon/Dockerfile.j2
@@ -3,5 +3,6 @@ MAINTAINER Kolla Project (https://launchpad.net/kolla)
 
 COPY start.sh /
 COPY config-external.sh /opt/kolla/
+COPY fetch_ceph_keys.py /usr/bin/
 
 CMD ["/start.sh"]
diff --git a/docker/ceph/ceph-mon/config-external.sh b/docker/ceph/ceph-mon/config-external.sh
index 75488239c..289cffc86 100644
--- a/docker/ceph/ceph-mon/config-external.sh
+++ b/docker/ceph/ceph-mon/config-external.sh
@@ -8,3 +8,33 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.mon.keyring"
+TARGET="/etc/ceph/ceph.client.mon.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.monmap"
+TARGET="/etc/ceph/ceph.monmap"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-mon/fetch_ceph_keys.py b/docker/ceph/ceph-mon/fetch_ceph_keys.py
new file mode 100755
index 000000000..ab2be9c16
--- /dev/null
+++ b/docker/ceph/ceph-mon/fetch_ceph_keys.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a stripped down version of an ansible module I wrote in Yaodu to
+# achieve the same goals we have for Kolla. I have relicensed it for Kolla
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp
+
+# Basically this module will fetch the admin and mon keyrings as well as the
+# monmap file. It then hashes the content, compresses them, and finally it
+# converts them to base64 to be safely transported around with ansible
+
+import base64
+import hashlib
+import json
+import os
+import sys
+import zlib
+
+
+def json_exit(msg=None, failed=False, changed=False):
+    if type(msg) is not dict:
+        msg = {'msg': str(msg)}
+    msg.update({'failed': failed, 'changed': changed})
+    print(json.dumps(msg))
+    sys.exit()
+
+
+def read_file(filename):
+    filename_path = os.path.join('/etc/ceph', filename)
+
+    if not os.path.exists(filename_path):
+        json_exit("file not found: {}".format(filename_path), failed=True)
+    if not os.access(filename_path, os.R_OK):
+        json_exit("file not readable: {}".format(filename_path), failed=True)
+
+    with open(filename_path, 'rb') as f:
+        raw_data = f.read()
+
+    return {'content': base64.b64encode(zlib.compress(raw_data)),
+            'sha1': hashlib.sha1(raw_data).hexdigest(),
+            'filename': filename}
+
+
+def main():
+    admin_keyring = 'ceph.client.admin.keyring'
+    mon_keyring = 'ceph.client.mon.keyring'
+    monmap = 'ceph.monmap'
+
+    files = [admin_keyring, mon_keyring, monmap]
+    json_exit({filename: read_file(filename) for filename in files})
+
+
+if __name__ == '__main__':
+    main()
diff --git a/docker/ceph/ceph-mon/start.sh b/docker/ceph/ceph-mon/start.sh
index eb5d7f5af..3afaa3ad4 100644..100755
--- a/docker/ceph/ceph-mon/start.sh
+++ b/docker/ceph/ceph-mon/start.sh
@@ -3,11 +3,11 @@
 set -o errexit
 
 CMD="/usr/bin/ceph-mon"
-ARGS="-d -i ${MON_NAME} --public-addr ${MON_IP}:6789"
+ARGS="-d -i $(hostname) --public-addr ${MON_IP}:6789"
 
 # Setup common paths
-KEYRING_ADMIN="/etc/ceph/ceph.admin.keyring"
-KEYRING_MON="/etc/ceph/ceph.mon.keyring"
+KEYRING_ADMIN="/etc/ceph/ceph.client.admin.keyring"
+KEYRING_MON="/etc/ceph/ceph.client.mon.keyring"
 MONMAP="/etc/ceph/ceph.monmap"
 MON_DIR="/var/lib/ceph/mon/ceph-$(hostname)"
 
@@ -21,7 +21,7 @@ set_configs
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     # Lookup our fsid from the ceph.conf
-    FSID="$(awk '/^fsid/ {print $3; exit}' ${ceph_conf})"
+    FSID="$(awk '/^fsid/ {print $3; exit}' /etc/ceph/ceph.conf)"
 
     # Generating initial keyrings and monmap
     ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *'
@@ -29,8 +29,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}"
     monmaptool --create --add "$(hostname)" "${MON_IP}" --fsid "${FSID}" "${MONMAP}"
 
-    # TODO(SamYaple): Return json parsible output to ansible
-    exit 0
+    echo "Sleeping until keys are fetched"
+    /bin/sleep infinity
 fi
 
 # This section runs on every mon that does not have a keyring already.
diff --git a/docker/ceph/ceph-osd/config-external.sh b/docker/ceph/ceph-osd/config-external.sh
index 8e17487ed..8928f91cb 100644
--- a/docker/ceph/ceph-osd/config-external.sh
+++ b/docker/ceph/ceph-osd/config-external.sh
@@ -8,3 +8,13 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-osd/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-osd/start.sh b/docker/ceph/ceph-osd/start.sh
index cf9114a62..84549ae8c 100644..100755
--- a/docker/ceph/ceph-osd/start.sh
+++ b/docker/ceph/ceph-osd/start.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
+set -o xtrace
 set -o errexit
 
-CMD="/usr/bin/ceph-osd"
-ARGS="-f -d -i ${OSD_ID} --osd-journal ${OSD_DIR}/journal -k ${OSD_DIR}/keyring"
-
 # Loading common functions.
 source /opt/kolla/kolla-common.sh
 
@@ -13,23 +11,45 @@ set_configs
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    # Creating a new label for the disk
-    parted "${OSD_DEV}" -s -- mklabel gpt
+    JOURNAL_PARTUUID=$(uuidgen)
+
+    # Formating disk for ceph
+    sgdisk --zap-all -- "${OSD_DEV}"
+    sgdisk --new=2:1M:5G --change-name=2:KOLLA_CEPH_JOURNAL --typecode=2:45B0969E-9B03-4F30-B4C6-B4B80CEFF106 --partition-guid=2:${JOURNAL_PARTUUID} --mbrtogpt -- "${OSD_DEV}"
+    sgdisk --largest-new=1 --change-name=1:KOLLA_CEPH_DATA --typecode=1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -- "${OSD_DEV}"
+    # This command may throw errors that we can safely ignore
+    partprobe || true
+
+    # We look up the appropriate device path with partition.
+    OSD_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?1)"
+    JOURNAL_PARTITION="${OSD_PARTITION%?}2"
 
-    # Preparing the OSD for use with Ceph
-    ceph-disk prepare "${OSD_DEV}"
     OSD_ID="$(ceph osd create)"
     OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
     mkdir -p "${OSD_DIR}"
-    mount "${OSD_DEV}1" "${OSD_DIR}"
-    ceph-osd -i "${OSD_ID}" --mkfs --mkkey
-    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow proflie osd' -i "${OSD_DIR}/keyring"
+    mkfs.xfs -f "${OSD_PARTITION}"
+    mount "${OSD_PARTITION}" "${OSD_DIR}"
 
-    # Adding osd to crush map
+    # This will through an error about no key existing. That is normal. It then
+    # creates the key in the next step.
+    ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey
+    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
+    umount "${OSD_PARTITION}"
+
+    # These commands only need to be run once per host but are safe to run
+    # repeatedly. This can be improved later or if any problems arise.
     ceph osd crush add-bucket "$(hostname)" host
     ceph osd crush move "$(hostname)" root=default
+
+    # Adding osd to crush map
     ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="$(hostname)"
     exit 0
 fi
 
+# We look up the appropriate journal since we cannot rely on symlinks
+JOURNAL_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?2)"
+OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
+CMD="/usr/bin/ceph-osd"
+ARGS="-f -d -i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
+
 exec $CMD $ARGS