Allow Kolla Ceph to deploy bluestore OSDs in Kolla

Support Kolla Ceph to deploy blustore OSDs. With the patch, Kolla
Ceph can deploy bluestore OSDs on ONE, TWO or THREE storage
devices.

Before deploy bluestore OSD, please prepare devices. The detailed
Please refer to [1] for devices initialization.

extend_start.sh: initialize and start bluestore OSD

find_disk.py: search the devices for bluestore OSD

[1]: specs/kolla-ceph-bluestore.rst

Partially-Implements: blueprint kolla-ceph-bluestore

Change-Id: I832f490de63e1aeb68814697cda610a51b622c1f
Signed-off-by: Tone Zhang <tone.zhang@arm.com>

1 files changed, 89 insertions, 21 deletions

diff --git a/docker/ceph/ceph-osd/extend_start.sh b/docker/ceph/ceph-osd/extend_start.sh
index ee745d0af..57ec84f4e 100644
--- a/docker/ceph/ceph-osd/extend_start.sh
+++ b/docker/ceph/ceph-osd/extend_start.sh
@@ -13,38 +13,97 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     # NOTE(SamYaple): Static gpt partcodes
     CEPH_JOURNAL_TYPE_CODE="45B0969E-9B03-4F30-B4C6-B4B80CEFF106"
     CEPH_OSD_TYPE_CODE="4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D"
+    CEPH_OSD_BS_WAL_TYPE_CODE="0FC63DAF-8483-4772-8E79-3D69D8477DE4"
+    CEPH_OSD_BS_DB_TYPE_CODE="CE8DF73C-B89D-45B0-AD98-D45332906d90"
 
     # Wait for ceph quorum before proceeding
     ceph quorum_status
 
     if [[ "${USE_EXTERNAL_JOURNAL}" == "False" ]]; then
         # Formatting disk for ceph
-        sgdisk --zap-all -- "${OSD_DEV}"
-        sgdisk --new=2:1M:5G -- "${JOURNAL_DEV}"
-        sgdisk --largest-new=1 -- "${OSD_DEV}"
+        if [[ "${OSD_STORETYPE}" == "bluestore" ]]; then
+            if [ -n "${OSD_BS_BLOCK_DEV}" ]; then
+                sgdisk --zap-all -- "${OSD_BS_BLOCK_DEV}"
+                sgdisk --new=1:0:+100M --mbrtogpt -- "${OSD_BS_BLOCK_DEV}"
+                sgdisk --largest-new=2 --mbrtogpt -- "${OSD_BS_BLOCK_DEV}"
+                sgdisk --zap-all -- "${OSD_BS_BLOCK_DEV}"2
+            fi
+
+            if [ -n "${OSD_BS_WAL_DEV}" ] && [ "${OSD_BS_BLOCK_DEV}" != "${OSD_BS_WAL_DEV}" ] && [ -n "${OSD_BS_WAL_PARTNUM}" ]; then
+                sgdisk --zap-all -- "${OSD_BS_WAL_DEV}""${OSD_BS_WAL_PARTNUM}"
+            fi
+
+            if [ -n "${OSD_BS_DB_DEV}" ] && [ "${OSD_BS_BLOCK_DEV}" != "${OSD_BS_DB_DEV}" ] && [ -n "${OSD_BS_DB_PARTNUM}" ]; then
+                sgdisk --zap-all -- "${OSD_BS_DB_DEV}""${OSD_BS_DB_PARTNUM}"
+            fi
+        else
+            sgdisk --zap-all -- "${OSD_DEV}"
+            sgdisk --new=2:1M:5G -- "${JOURNAL_DEV}"
+            sgdisk --largest-new=1 -- "${OSD_DEV}"
+        fi
         # NOTE(SamYaple): This command may throw errors that we can safely ignore
         partprobe || true
 
     fi
 
-    OSD_ID=$(ceph osd create)
-    OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
-    mkdir -p "${OSD_DIR}"
+    if [[ "${OSD_STORETYPE}" == "bluestore" ]]; then
+        OSD_UUID=$(uuidgen)
+        OSD_ID=$(ceph osd new "${OSD_UUID}")
+        OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
+        mkdir -p "${OSD_DIR}"
+
+        mkfs.xfs -f "${OSD_BS_BLOCK_DEV}"1
+        mount "${OSD_BS_BLOCK_DEV}"1 "${OSD_DIR}"
+
+        # This will through an error about no key existing. That is normal. It then
+        # creates the key in the next step.
+        ceph-osd -i "${OSD_ID}" --mkkey
+        echo "bluestore" > "${OSD_DIR}"/type
+        sgdisk "--change-name=2:KOLLA_CEPH_DATA_BS_B_${OSD_ID}" "--typecode=2:${CEPH_OSD_TYPE_CODE}" -- "${OSD_BS_BLOCK_DEV}"
+
+        if [ -n "${OSD_BS_WAL_DEV}" ] && [ "${OSD_BS_BLOCK_DEV}" != "${OSD_BS_WAL_DEV}" ] && [ -n "${OSD_BS_WAL_PARTNUM}" ]; then
+            sgdisk "--change-name="${OSD_BS_WAL_PARTNUM}":KOLLA_CEPH_DATA_BS_W_${OSD_ID}" "--typecode=1:${CEPH_OSD_BS_WAL_TYPE_CODE}" -- "${OSD_BS_WAL_DEV}"
+        fi
+
+        if [ -n "${OSD_BS_DB_DEV}" ] && [ "${OSD_BS_BLOCK_DEV}" != "${OSD_BS_DB_DEV}" ] && [ -n "${OSD_BS_DB_PARTNUM}" ]; then
+            sgdisk "--change-name="${OSD_BS_DB_PARTNUM}":KOLLA_CEPH_DATA_BS_D_${OSD_ID}" "--typecode=1:${CEPH_OSD_BS_DB_TYPE_CODE}" -- "${OSD_BS_DB_DEV}"
+        fi
+
+        partprobe || true
+
+        ln -sf /dev/disk/by-partlabel/KOLLA_CEPH_DATA_BS_B_"${OSD_ID}" "${OSD_DIR}"/block
 
-    if [[ "${OSD_FILESYSTEM}" == "btrfs" ]]; then
-        mkfs.btrfs -f "${OSD_PARTITION}"
-    elif [[ "${OSD_FILESYSTEM}" == "ext4" ]]; then
-        mkfs.ext4 "${OSD_PARTITION}"
+        if [ -n "${OSD_BS_WAL_DEV}" ] && [ "${OSD_BS_BLOCK_DEV}" != "${OSD_BS_WAL_DEV}" ] && [ -n "${OSD_BS_WAL_PARTNUM}" ]; then
+            ln -sf /dev/disk/by-partlabel/KOLLA_CEPH_DATA_BS_W_"${OSD_ID}" "${OSD_DIR}"/block.wal
+        fi
+
+        if [ -n "${OSD_BS_DB_DEV}" ] && [ "${OSD_BS_BLOCK_DEV}" != "${OSD_BS_DB_DEV}" ]i && [ -n "${OSD_BS_DB_PARTNUM}" ]; then
+            ln -sf /dev/disk/by-partlabel/KOLLA_CEPH_DATA_BS_D_"${OSD_ID}" "${OSD_DIR}"/block.db
+        fi
+
+        ceph-osd -i "${OSD_ID}" --mkfs -k "${OSD_DIR}"/keyring --osd-uuid "${OSD_UUID}"
+        ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
+        umount "${OSD_BS_BLOCK_DEV}"1
     else
-        mkfs.xfs -f "${OSD_PARTITION}"
-    fi
-    mount "${OSD_PARTITION}" "${OSD_DIR}"
+        OSD_ID=$(ceph osd create)
+        OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
+        mkdir -p "${OSD_DIR}"
+
+        if [[ "${OSD_FILESYSTEM}" == "btrfs" ]]; then
+            mkfs.btrfs -f "${OSD_PARTITION}"
+        elif [[ "${OSD_FILESYSTEM}" == "ext4" ]]; then
+            mkfs.ext4 "${OSD_PARTITION}"
+        else
+            mkfs.xfs -f "${OSD_PARTITION}"
+        fi
+        mount "${OSD_PARTITION}" "${OSD_DIR}"
 
-    # This will through an error about no key existing. That is normal. It then
-    # creates the key in the next step.
-    ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey
-    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
-    umount "${OSD_PARTITION}"
+        # This will through an error about no key existing. That is normal. It then
+        # creates the key in the next step.
+        ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey
+        ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
+        umount "${OSD_PARTITION}"
+    fi
 
     if [[ "${!CEPH_CACHE[@]}" ]]; then
         CEPH_ROOT_NAME=cache
@@ -63,11 +122,20 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="${HOSTNAME}${CEPH_ROOT_NAME:+-${CEPH_ROOT_NAME}}"
 
     # Setting partition name based on ${OSD_ID}
-    sgdisk "--change-name=${OSD_PARTITION_NUM}:KOLLA_CEPH_DATA_${OSD_ID}" "--typecode=${OSD_PARTITION_NUM}:${CEPH_OSD_TYPE_CODE}" -- "${OSD_DEV}"
-    sgdisk "--change-name=${JOURNAL_PARTITION_NUM}:KOLLA_CEPH_DATA_${OSD_ID}_J" "--typecode=${JOURNAL_PARTITION_NUM}:${CEPH_JOURNAL_TYPE_CODE}" -- "${JOURNAL_DEV}"
+    if [[ "${OSD_STORETYPE}" == "bluestore" ]]; then
+        sgdisk "--change-name=1:KOLLA_CEPH_BSDATA_${OSD_ID}" -- "${OSD_BS_BLOCK_DEV}"
+    else
+        sgdisk "--change-name=${OSD_PARTITION_NUM}:KOLLA_CEPH_DATA_${OSD_ID}" "--typecode=${OSD_PARTITION_NUM}:${CEPH_OSD_TYPE_CODE}" -- "${OSD_DEV}"
+        sgdisk "--change-name=${JOURNAL_PARTITION_NUM}:KOLLA_CEPH_DATA_${OSD_ID}_J" "--typecode=${JOURNAL_PARTITION_NUM}:${CEPH_JOURNAL_TYPE_CODE}" -- "${JOURNAL_DEV}"
+    fi
+    partprobe || true
 
     exit 0
 fi
 
 OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
-ARGS="-i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
+if [[ "${OSD_STORETYPE}" == "bluestore" ]]; then
+    ARGS="-i ${OSD_ID}"
+else
+    ARGS="-i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
+fi