External Ceph - Implementation Cinder

This patch adds support for external Ceph clusters for Cinder.

For clean integration the backend configuration mechanism had to be
slightly adjusted.

We now have the option to enable multiple backends for Cinder
independently.

Currently, the flags cinder_backend_iscsi and cinder_backend_ceph are
used to toggle backends.

Documentation on how to use external ceph was added.

Change-Id: I7e0267b90d62d6d881f24f063cdb894422ec8618
Partially-Implements: Blueprint: external-ceph

11 files changed, 234 insertions, 46 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 911dc5e68..9562236eb 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -244,13 +244,15 @@ kibana_user: "kibana"
 
 
 #################################
-# Cinder - Block Storage options
+# Cinder options
 #################################
-cinder_volume_driver: "{{ 'ceph' if enable_ceph | bool else 'lvm' }}"
-cinder_volume_group: "{{ 'cinder-volumes' if enable_iscsi | bool else '' }}"
-cinder_volume_backend_name: "{{ 'cinder-volumes' if enable_iscsi | bool else '' }}"
-cinder_iscsi_helper: "{{ 'tgtadm' if enable_iscsi | bool else '' }}"
-cinder_iscsi_protocol: "{{ 'iscsi' if enable_iscsi | bool else '' }}"
+cinder_backend_iscsi: "{{ enable_iscsi }}"
+cinder_backend_ceph: "{{ enable_ceph }}"
+cinder_volume_driver: "{{ 'lvm' if cinder_backend_iscsi }}"
+cinder_volume_group: "{{ 'cinder-volumes' if cinder_backend_iscsi | bool else '' }}"
+cinder_volume_backend_name: "{{ 'cinder-volumes' if cinder_backend_iscsi | bool else '' }}"
+cinder_iscsi_helper: "{{ 'tgtadm' if cinder_backend_iscsi | bool else '' }}"
+cinder_iscsi_protocol: "{{ 'iscsi' if cinder_backend_iscsi | bool else '' }}"
 
 ###################
 # Ceph options
diff --git a/ansible/roles/cinder/tasks/bootstrap.yml b/ansible/roles/cinder/tasks/bootstrap.yml
index 7762f2892..4080018b3 100644
--- a/ansible/roles/cinder/tasks/bootstrap.yml
+++ b/ansible/roles/cinder/tasks/bootstrap.yml
@@ -45,11 +45,11 @@
     action: "create_volume"
     common_options: "{{ docker_common_options }}"
     name: "cinder"
-  when: "{{ enable_iscsi | bool }}"
+  when: "{{ cinder_backend_iscsi | bool }}"
 
 - name: Creating iscsi info volume
   kolla_docker:
     action: "create_volume"
     common_options: "{{ docker_common_options }}"
     name: "iscsi_info"
-  when: "{{ enable_iscsi | bool }}"
+  when: "{{ cinder_backend_iscsi | bool }}"
diff --git a/ansible/roles/cinder/tasks/deploy.yml b/ansible/roles/cinder/tasks/deploy.yml
index e9502bee2..b01e2efae 100644
--- a/ansible/roles/cinder/tasks/deploy.yml
+++ b/ansible/roles/cinder/tasks/deploy.yml
@@ -1,13 +1,19 @@
 ---
 - include: ceph.yml
   when:
-    - enable_ceph | bool
+    - (enable_ceph | bool) and (cinder_backend_ceph | bool)
     - inventory_hostname in groups['ceph-mon'] or
         inventory_hostname in groups['cinder-api'] or
         inventory_hostname in groups['cinder-volume'] or
         inventory_hostname in groups['cinder-scheduler'] or
         inventory_hostname in groups['cinder-backup']
 
+- include: external_ceph.yml
+  when:
+    - (enable_ceph | bool == False) and (cinder_backend_ceph | bool)
+    - inventory_hostname in groups['cinder-volume'] or
+      inventory_hostname in groups['cinder-backup']
+
 - include: register.yml
   when: inventory_hostname in groups['cinder-api']
 
diff --git a/ansible/roles/cinder/tasks/external_ceph.yml b/ansible/roles/cinder/tasks/external_ceph.yml
new file mode 100644
index 000000000..45fe480ee
--- /dev/null
+++ b/ansible/roles/cinder/tasks/external_ceph.yml
@@ -0,0 +1,36 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/cinder-{{ item }}"
+    state: "directory"
+  when: inventory_hostname in groups['cinder-volume'] or
+        inventory_hostname in groups['cinder-backup']
+  with_items:
+    - volume
+    - backup
+
+- name: Copying over ceph.conf for Cinder
+  merge_configs:
+    vars:
+      service_name: "{{ item }}"
+    sources:
+      - "{{ node_custom_config }}/cinder/ceph.conf"
+      - "{{ node_custom_config }}/cinder/{{ item }}/ceph.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
+  with_items:
+    - "cinder-backup"
+    - "cinder-volume"
+
+- name: Copy over Ceph keyring files for cinder-volume
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/cinder-volume/"
+  with_fileglob:
+    - "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"
+
+- name: Copy over Ceph keyring files for cinder-backup
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/cinder-backup/"
+  with_fileglob:
+    - "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"
diff --git a/ansible/roles/cinder/tasks/start.yml b/ansible/roles/cinder/tasks/start.yml
index 7f433646f..0047d0bba 100644
--- a/ansible/roles/cinder/tasks/start.yml
+++ b/ansible/roles/cinder/tasks/start.yml
@@ -30,8 +30,8 @@
       - "/etc/localtime:/etc/localtime:ro"
       - "/dev/:/dev/"
       - "/run/:/run/"
-      - "{% if enable_iscsi | bool %}cinder:/var/lib/cinder{% endif %}"
-      - "{% if enable_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
+      - "{% if cinder_backend_iscsi | bool %}cinder:/var/lib/cinder{% endif %}"
+      - "{% if cinder_backend_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
       - "kolla_logs:/var/log/kolla/"
 
 - name: Starting cinder-volume container
diff --git a/ansible/roles/cinder/templates/cinder-backup.json.j2 b/ansible/roles/cinder/templates/cinder-backup.json.j2
index 33929eca4..fefb9d9c5 100644
--- a/ansible/roles/cinder/templates/cinder-backup.json.j2
+++ b/ansible/roles/cinder/templates/cinder-backup.json.j2
@@ -6,27 +6,13 @@
             "dest": "/etc/cinder/cinder.conf",
             "owner": "cinder",
             "perm": "0600"
-        },
+        }{% if cinder_backend_ceph | bool %},
         {
-            "source": "{{ container_config_directory }}/ceph.client.cinder-backup.keyring",
-            "dest": "/etc/ceph/ceph.client.cinder-backup.keyring",
+            "source": "{{ container_config_directory }}/ceph.*",
+            "dest": "/etc/ceph/",
             "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        },
-        {
-            "source": "{{ container_config_directory }}/ceph.client.cinder.keyring",
-            "dest": "/etc/ceph/ceph.client.cinder.keyring",
-            "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        },
-        {
-            "source": "{{ container_config_directory }}/ceph.conf",
-            "dest": "/etc/ceph/ceph.conf",
-            "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        }
+            "perm": "0700",
+            "optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
+        }{% endif %}
     ]
 }
diff --git a/ansible/roles/cinder/templates/cinder-volume.json.j2 b/ansible/roles/cinder/templates/cinder-volume.json.j2
index 549528410..e82912ad0 100644
--- a/ansible/roles/cinder/templates/cinder-volume.json.j2
+++ b/ansible/roles/cinder/templates/cinder-volume.json.j2
@@ -6,20 +6,18 @@
             "dest": "/etc/cinder/cinder.conf",
             "owner": "cinder",
             "perm": "0600"
-        },
+        }{% if cinder_backend_ceph | bool %},
         {
-            "source": "{{ container_config_directory }}/ceph.client.cinder.keyring",
-            "dest": "/etc/ceph/ceph.client.cinder.keyring",
+            "source": "{{ container_config_directory }}/ceph.*",
+            "dest": "/etc/ceph/",
             "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
+            "perm": "0700"
         },
         {
             "source": "{{ container_config_directory }}/ceph.conf",
             "dest": "/etc/ceph/ceph.conf",
             "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        }
+            "perm": "0600"
+        }{% endif %}
     ]
 }
diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2
index 86f7375a3..642225ebd 100644
--- a/ansible/roles/cinder/templates/cinder.conf.j2
+++ b/ansible/roles/cinder/templates/cinder.conf.j2
@@ -16,14 +16,14 @@ glance_api_version = 2
 
 os_region_name = {{ openstack_region_name }}
 
-{% if cinder_volume_driver == "lvm" %}
-enabled_backends = {{ cinder_volume_backend_name }}
-{% elif cinder_volume_driver == "ceph" %}
+{% if enable_ceph | bool and cinder_backend_ceph | bool %}
 default_volume_type = rbd-1
 enabled_backends = rbd-1
+{% elif cinder_backend_iscsi | bool %}
+enabled_backends = {{ cinder_volume_backend_name }}
 {% endif %}
 
-{% if service_name == "cinder-backup" and cinder_volume_driver == "ceph" %}
+{% if service_name == "cinder-backup" and enable_ceph | bool and cinder_backend_ceph | bool %}
 backup_driver = cinder.backup.drivers.ceph
 backup_ceph_conf = /etc/ceph/ceph.conf
 backup_ceph_user = cinder-backup
@@ -76,7 +76,7 @@ rabbit_ha_queues = true
 rabbit_hosts = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
 
 
-{% if cinder_volume_driver == "lvm" %}
+{% if cinder_backend_iscsi | bool %}
 [{{ cinder_volume_backend_name }}]
 volume_group = {{ cinder_volume_group }}
 volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
@@ -84,7 +84,8 @@ volume_backend_name = {{ cinder_volume_backend_name }}
 iscsi_ip_address = {{ hostvars[groups['tgtd'][0]]['ansible_' + hostvars[groups['tgtd'][0]]['storage_interface']]['ipv4']['address'] }}
 iscsi_helper = {{ cinder_iscsi_helper }}
 iscsi_protocol = {{ cinder_iscsi_protocol }}
-{% elif cinder_volume_driver == "ceph" %}
+
+{% elif enable_ceph | bool and cinder_backend_ceph | bool %}
 [rbd-1]
 volume_driver = cinder.volume.drivers.rbd.RBDDriver
 rbd_pool = {{ ceph_cinder_pool_name }}
diff --git a/doc/external-ceph-guide.rst b/doc/external-ceph-guide.rst
new file mode 100644
index 000000000..230919163
--- /dev/null
+++ b/doc/external-ceph-guide.rst
@@ -0,0 +1,153 @@
+.. _external-ceph-guide:
+
+=============
+External Ceph
+=============
+
+Sometimes it is necessary to connect OpenStack services to an existing Ceph
+cluster instead of deploying it with Kolla. This can be achieved with only a
+few configuration steps in Kolla.
+
+Requirements
+============
+
+* An existing installation of Ceph
+* Existing Ceph storage pools
+* Existing credentials in Ceph for OpenStack services to connect to Ceph
+(Glance, Cinder)
+
+Enabling External Ceph
+======================
+
+Using external Ceph with Kolla means not to deploy Ceph via Kolla. Therefore,
+disable Ceph deployment in ``/etc/kolla/global.yml``
+
+::
+
+  enable_ceph: "no"
+
+There are flags indicating individual services to use ceph or not which default
+to the value of ``enable_ceph``. Those flags now need to be activated in order
+to activate external Ceph integration. This can be done individually per
+service in ``/etc/kolla/global.yml``:
+
+::
+
+  glance_enable_ceph: "yes"
+  cinder_backend_ceph: "yes"
+
+The combination of ``enable_ceph: "no"`` and ``<service>_enable_ceph: "yes"``
+triggers the activation of external ceph mechanism in Kolla.
+
+Configuring External Ceph
+=========================
+
+Glance
+------
+
+Configuring Glance for Ceph includes three steps:
+
+1) Configure RBD backend in glance-api.conf
+2) Create Ceph configuration file in /etc/ceph/ceph.conf
+3) Create Ceph keyring file in /etc/ceph/ceph.client.<username>.keyring
+
+Step 1 is done by using Kolla's INI merge mechanism: Create a file in
+``/etc/kolla/config/glance/glance-api.conf`` with the following contents:
+
+::
+
+  [DEFAULT]
+  show_image_direct_url = True
+
+  [glance_store]
+  stores = rbd
+  default_store = rbd
+  rbd_store_chunk_size = 8
+  rbd_store_pool = images
+  rbd_store_user = glance
+  rbd_store_ceph_conf = /etc/ceph/ceph.conf
+  rados_connect_timeout = 0
+
+  [image_format]
+  container_formats = bare
+  disk_formats = raw
+
+Now put ceph.conf and the keyring file (name depends on the username created in
+Ceph) into the same directory, for example:
+
+/etc/kolla/config/glance/ceph.conf
+
+::
+
+  [global]
+  fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
+  mon_initial_members = ceph-0
+  mon_host = 192.168.0.56
+  auth_cluster_required = cephx
+  auth_service_required = cephx
+  auth_client_required = cephx
+
+/etc/kolla/config/glance/ceph.client.glance.keyring
+
+::
+
+  [client.glance]
+  key = AQAg5YRXS0qxLRAAXe6a4R1a15AoRx7ft80DhA==
+
+Kolla will pick up all files named ceph.* in this directory an copy them to the
+/etc/ceph/ directory of the container.
+
+Cinder
+------
+
+Configuring external Ceph for Cinder works very similar to
+Glance. The required Cinder configuration goes into
+/etc/kolla/config/cinder/cinder-volume.conf:
+
+::
+
+  [DEFAULT]
+  enabled_backends=rbd-1
+
+  [rbd-1]
+  rbd_ceph_conf=/etc/ceph/ceph.conf
+  rbd_user=cinder
+  backend_host=rbd:volumes
+  rbd_pool=volumes
+  volume_backend_name=rbd-1
+  volume_driver=cinder.volume.drivers.rbd.RBDDriver
+
+Next, place the ceph.conf file into
+/etc/kolla/config/cinder/ceph.conf:
+
+::
+
+  [global]
+  fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
+  mon_initial_members = ceph-0
+  mon_host = 192.168.0.56
+  auth_cluster_required = cephx
+  auth_service_required = cephx
+  auth_client_required = cephx
+
+Separate configuration options can be configured for
+cinder-volume and cinder-backup by adding ceph.conf files to
+/etc/kolla/config/cinder/cinder-volume and
+/etc/kolla/config/cinder/cinder-backup respectively. They
+will be merged with /etc/kolla/config/cinder/ceph.conf.
+
+Ceph keyrings are deployed per service and placed into
+cinder-volume and cinder-backup directories:
+
+::
+
+  root@deploy:/etc/kolla/config# cat
+  cinder/cinder-backup/ceph.client.cinder.keyring
+  [client.cinder]
+          key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
+  root@deploy:/etc/kolla/config# cat
+  cinder/cinder-volume/ceph.client.cinder.keyring
+  [client.cinder]
+          key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
+
+It is important that the files are named ceph.client*.
diff --git a/doc/index.rst b/doc/index.rst
index 461829ac7..a5d50bc6c 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -51,6 +51,7 @@ Kolla Services
    :maxdepth: 1
 
    ceph-guide
+   external-ceph-guide
    cinder-guide
    ironic-guide
    manila-guide
diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml
index 42f725dca..cd4c0815b 100644
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@@ -129,6 +129,7 @@ neutron_external_interface: "eth1"
 # when ceph is not deployed by Kolla.
 #glance_enable_ceph: "{{ enable_ceph }}"
 
+
 ###################
 # Ceph options
 ###################
@@ -144,8 +145,12 @@ neutron_external_interface: "eth1"
 
 
 #######################
-# Cinder iSCSI options
+# Cinder options
 #######################
+# Enable / disable Cinder backends
+cinder_backend_iscsi: "{{ enable_iscsi }}"
+cinder_backend_ceph: "{{ enable_ceph }}"
+
 # Cinder's iSCSI backend !!!REQUIRES!!! two parameters:
 # 1 - IP address of the server hosting LVM Volume group
 # 2 - The name of Volume group which Cinder will use.