CI: Move from ceph-ansible to cephadm

Change-Id: I81a4f8f8b8faa7559740531bb16d8aec7fc23f9b

5 files changed, 192 insertions, 0 deletions

diff --git a/roles/cephadm/defaults/main.yml b/roles/cephadm/defaults/main.yml
new file mode 100644
index 000000000..abae1377b
--- /dev/null
+++ b/roles/cephadm/defaults/main.yml
@@ -0,0 +1,29 @@
+---
+cephadm_ceph_apt_repo: "deb http://download.ceph.com/debian-octopus/ {{ ansible_distribution_release }} main"
+
+cephadm_ceph_rpm_repos:
+  - shortname: "ceph"
+    name: "Ceph {{ ansible_architecture }} repository"
+    url: "http://download.ceph.com/rpm-octopus/el8/{{ ansible_architecture }}/"
+  - shortname: "ceph-noarch"
+    name: "Ceph noarch repository"
+    url: "http://download.ceph.com/rpm-octopus/el8/noarch/"
+
+cephadm_ceph_rpm_gpg_key: http://download.ceph.com/keys/release.gpg
+
+cephadm_ceph_osd_devices:
+  - primary:/dev/cephvg/cephlv
+  - secondary1:/dev/cephvg/cephlv
+  - secondary2:/dev/cephvg/cephlv
+
+cephadm_ceph_pools:
+  - backup
+  - images
+  - vms
+  - volumes
+
+cephadm_ceph_users:
+  - client.glance mon 'profile rbd' osd 'profile rbd pool=images' mgr 'profile rbd pool=images' -o /var/run/ceph/ceph.client.glance.keyring
+  - client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms' -o /var/run/ceph/ceph.client.cinder.keyring
+  - client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups' -o /var/run/ceph/ceph.client.cinder-backup.keyring
+  - client.nova mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms' -o /var/run/ceph/ceph.client.nova.keyring
diff --git a/roles/cephadm/tasks/main.yml b/roles/cephadm/tasks/main.yml
new file mode 100644
index 000000000..d8795050e
--- /dev/null
+++ b/roles/cephadm/tasks/main.yml
@@ -0,0 +1,113 @@
+---
+- include_tasks: pkg_{{ ansible_os_family | lower }}.yml
+
+- name: Install cephadm
+  package:
+    name: cephadm
+    state: present
+  become: True
+
+- name: Ensure /etc/ceph exists
+  file:
+    path: /etc/ceph
+    state: directory
+  become: True
+
+- name: Generate ssh key for cephadm
+  openssh_keypair:
+    path: "/etc/ceph/cephadm.id"
+    size: 4096
+    comment: "cephadm"
+  become: True
+  register: cephadm_ssh_key
+
+- name: Save public key
+  copy:
+    content: "{{ cephadm_ssh_key.public_key }}"
+    dest: /etc/ceph/cephadm.pub
+  become: True
+
+- name: Copy cephadm public key to all hosts
+  authorized_key:
+    user: root
+    state: present
+    key: "{{ cephadm_ssh_key.public_key }}"
+  become: True
+  with_inventory_hostnames:
+    - all
+  delegate_to: "{{ item }}"
+
+- name: Bootstrap cephadm
+  vars:
+    mon_ip: "{{ hostvars[inventory_hostname]['ansible_'+api_interface_name].ipv4.address }}"
+  command:
+    cmd: >
+         cephadm bootstrap
+         --ssh-private-key=/etc/ceph/cephadm.id
+         --ssh-public-key=/etc/ceph/cephadm.pub
+         --skip-monitoring-stack
+         --skip-dashboard
+         --skip-firewalld
+         --mon-ip={{ mon_ip }}
+  become: True
+  register: cephadm_bootstrap_output
+
+- name: Get ceph fsid
+  vars:
+    regexp: 'Cluster fsid: (.*)'
+  set_fact:
+    ceph_fsid: "{{ cephadm_bootstrap_output.stdout | regex_search(regexp,'\\1') | first }}"
+
+- name: Template out cluster spec
+  template:
+    src: templates/cephadm.yml.j2
+    dest: "/var/run/ceph/{{ ceph_fsid }}/cluster.yml"
+  become: True
+
+- name: Apply cluster spec
+  command:
+    cmd: >
+         cephadm shell --
+         ceph orch apply -i /var/run/ceph/cluster.yml
+  become: True
+
+- name: Add osds
+  command:
+    cmd: >
+         cephadm shell --
+         ceph orch daemon add osd {{ item }}
+  become: True
+  loop: "{{ cephadm_ceph_osd_devices }}"
+
+- name: Create and initialise pools for OpenStack services
+  command:
+    cmd: >
+         cephadm shell --
+         ceph osd pool create {{ item }}
+  with_items: "{{ cephadm_ceph_pools }}"
+  become: true
+
+- name: Create users for OpenStack services
+  command:
+    cmd: >
+         cephadm shell --
+         ceph auth get-or-create {{ item }}
+  become: true
+  with_items: "{{ cephadm_ceph_users }}"
+
+# TODO(mnasiadka): Fix merge_configs to support tabs
+- name: Generate ceph.conf without tabs
+  vars:
+    ceph_conf_fixed: |
+      [global]
+      fsid = {{ ceph_fsid }}
+      mon_host = {% for host in groups['all'] %} {{ hostvars[host]['ansible_'+api_interface_name].ipv4.address }} {% if not loop.last %},{% endif %} {% endfor %}
+  copy:
+    content: "{{ ceph_conf_fixed }}"
+    dest: /etc/ceph/ceph.conf
+  become: True
+
+- name: Check ceph health
+  command:
+    cmd: cephadm shell -- ceph health detail
+  become: True
diff --git a/roles/cephadm/tasks/pkg_debian.yml b/roles/cephadm/tasks/pkg_debian.yml
new file mode 100644
index 000000000..d88875213
--- /dev/null
+++ b/roles/cephadm/tasks/pkg_debian.yml
@@ -0,0 +1,13 @@
+---
+- name: Ensure apt sources list directory exists
+  file:
+    path: /etc/apt/sources.list.d
+    state: directory
+    recurse: yes
+  become: True
+
+- name: Enable Ceph apt repository
+  apt_repository:
+    repo: "{{ cephadm_ceph_apt_repo }}"
+    filename: ceph
+  become: True
diff --git a/roles/cephadm/tasks/pkg_redhat.yml b/roles/cephadm/tasks/pkg_redhat.yml
new file mode 100644
index 000000000..b1d98b3bd
--- /dev/null
+++ b/roles/cephadm/tasks/pkg_redhat.yml
@@ -0,0 +1,16 @@
+---
+- name: Ensure yum repos directory exists
+  file:
+    path: /etc/yum.repos.d/
+    state: directory
+    recurse: yes
+  become: True
+
+- name: Enable Ceph base yum repository
+  yum_repository:
+    name: "{{ item.shortname }}"
+    description: "{{ item.name }}"
+    baseurl: "{{ item.url }}"
+    gpgkey: "{{ cephadm_ceph_rpm_gpg_key }}"
+  loop: "{{ cephadm_ceph_rpm_repos }}"
+  become: True
diff --git a/roles/cephadm/templates/cephadm.yml.j2 b/roles/cephadm/templates/cephadm.yml.j2
new file mode 100644
index 000000000..236ede26f
--- /dev/null
+++ b/roles/cephadm/templates/cephadm.yml.j2
@@ -0,0 +1,21 @@
+{% for host in groups['all'] %}
+---
+service_type: host
+hostname: {{ host }}
+labels:
+- mon
+- mgr
+- osd
+{% endfor %}
+---
+service_type: mon
+placement:
+  label: "mon"
+---
+service_type: mgr
+placement:
+  label: "mgr"
+---
+service_type: crash
+placement:
+  host_pattern: "*"