CI: Add Ceph-Ansible jobs

* Adding zuul centos-source/ubuntu-source ceph-ansible jobs
* Jobs will deploy all Ceph integrated OpenStack components, i.e.
  cinder, glance, nova
* Will utilize core openstack testing script

Depends-On: https://review.opendev.org/685032
Depends-On: https://review.opendev.org/698301

Implements: blueprint ceph-ansible
Change-Id: I233082b46785f74014177f579aeac887a25b2ae2

13 files changed, 201 insertions, 9 deletions

diff --git a/tests/check-config.sh b/tests/check-config.sh
index c8c172c7c..833746040 100755
--- a/tests/check-config.sh
+++ b/tests/check-config.sh
@@ -18,8 +18,10 @@ function check_config {
                 -not -path /etc/kolla \
                 -not -name admin-openrc.sh \
                 -not -name globals.yml \
+                -not -name ceph-ansible.yml \
                 -not -name header \
                 -not -name inventory \
+                -not -name ceph-inventory \
                 -not -name kolla-build.conf \
                 -not -name passwords.yml \
                 -not -name passwords.yml.old \
diff --git a/tests/deploy-ceph-ansible.sh b/tests/deploy-ceph-ansible.sh
new file mode 100755
index 000000000..9c4ef7297
--- /dev/null
+++ b/tests/deploy-ceph-ansible.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+function setup_ceph_ansible {
+    # FIXME(mnasiadka): Use python3 when we move to CentOS 8
+    # (there are no python3 selinux bindings for 3 on C7)
+    # see https://bugs.centos.org/view.php?id=16389
+
+    # Prepare virtualenv for ceph-ansible deployment
+    virtualenv --system-site-packages ~/ceph-venv
+    ~/ceph-venv/bin/pip install -Ir requirements.txt
+    ~/ceph-venv/bin/pip install -IU selinux
+}
+
+function deploy_ceph_ansible {
+    RAW_INVENTORY=/etc/kolla/ceph-inventory
+
+    . ~/ceph-venv/bin/activate
+
+    cp site-container.yml.sample site-container.yml
+    ansible-playbook -i ${RAW_INVENTORY} -e @/etc/kolla/ceph-ansible.yml -vvv site-container.yml --skip-tags=with_pkg &> /tmp/logs/ansible/deploy-ceph
+}
+
+setup_ceph_ansible
+deploy_ceph_ansible
diff --git a/tests/get_logs.sh b/tests/get_logs.sh
index b6e450db4..58a5e8b39 100644
--- a/tests/get_logs.sh
+++ b/tests/get_logs.sh
@@ -78,13 +78,20 @@ copy_logs() {
     (docker info && docker images && docker ps -a && docker network ls && docker inspect $(docker ps -aq)) > ${LOG_DIR}/system_logs/docker-info.txt
 
     # ceph related logs
-    if [[ $(docker ps --filter name=ceph_mon --format "{{.Names}}") ]]; then
-        docker exec ceph_mon ceph --connect-timeout 5 -s > ${LOG_DIR}/kolla/ceph/ceph_s.txt
+    # NOTE(mnasiadka): regex to match both ceph_mon and ceph-mon-$hostname
+    for container in $(docker ps --filter name=ceph.\?mon --format "{{.Names}}"); do
+        if [ $container == "ceph_mon" ]; then
+            CEPH_LOG_DIR="${LOG_DIR}/kolla/ceph"
+        else
+            CEPH_LOG_DIR="${LOG_DIR}/ceph"
+            mkdir -p ${CEPH_LOG_DIR}
+        fi
+        docker exec ${container} ceph --connect-timeout 5 -s > ${CEPH_LOG_DIR}/ceph_s.txt
         # NOTE(yoctozepto): osd df removed on purpose to avoid CI POST_FAILURE due to a possible hang:
         # as of ceph mimic it hangs when MON is operational but MGR not
         # its usefulness is mediocre and having POST_FAILUREs is bad
-        docker exec ceph_mon ceph --connect-timeout 5 osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt
-    fi
+        docker exec ${container} ceph --connect-timeout 5 osd tree > ${CEPH_LOG_DIR}/ceph_osd_tree.txt
+    done
 
     # bifrost related logs
     if [[ $(docker ps --filter name=bifrost_deploy --format "{{.Names}}") ]]; then
diff --git a/tests/run.yml b/tests/run.yml
index b77eeb520..ceff836b9 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -17,10 +17,11 @@
         kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
         kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
         infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
+        ceph_ansible_src_dir: "{{ ansible_env.PWD }}/src/github.com/ceph/ceph-ansible"
         need_build_image: false
         build_image_tag: "change_{{ zuul.change | default('none') }}"
         openstack_core_enabled: "{{ openstack_core_enabled }}"
-        openstack_core_tested: "{{ scenario in ['core', 'ceph', 'cinder-lvm', 'cells', 'swift'] }}"
+        openstack_core_tested: "{{ scenario in ['core', 'ceph', 'ceph-ansible', 'cinder-lvm', 'cells', 'swift'] }}"
         dashboard_enabled: "{{ openstack_core_enabled }}"
         # TODO(mgoddard): Remove when previous_release is ussuri.
         playbook_python_version: "{{ '2' if is_upgrade and previous_release == 'train' else '3' }}"
@@ -33,10 +34,10 @@
 
     - name: Prepare disks for a storage service
       script: "setup_disks.sh {{ disk_type }}"
-      when: scenario in ['cinder-lvm', 'ceph', 'zun', 'swift']
+      when: scenario in ['cinder-lvm', 'ceph', 'ceph-ansible', 'zun', 'swift']
       become: true
       vars:
-        disk_type: "{{ ceph_storetype if scenario == 'ceph' else scenario }}"
+        disk_type: "{{ ceph_storetype if scenario in ['ceph', 'ceph-ansible'] else scenario }}"
         ceph_storetype: "{{ hostvars[inventory_hostname].get('ceph_osd_storetype') }}"
 
 - hosts: primary
@@ -132,6 +133,14 @@
         - src: "tests/templates/ironic-overrides.j2"
           dest: /etc/kolla/config/ironic.conf
           when: "{{ scenario == 'ironic' }}"
+        # Ceph-Ansible inventory
+        - src: "tests/templates/ceph-inventory.j2"
+          dest: /etc/kolla/ceph-inventory
+          when: "{{ scenario == 'ceph-ansible' }}"
+        # ceph-ansible.yml
+        - src: "tests/templates/ceph-ansible.j2"
+          dest: /etc/kolla/ceph-ansible.yml
+          when: "{{ scenario == 'ceph-ansible' }}"
       when: item.when | default(true)
 
     - block:
@@ -257,6 +266,46 @@
     # ready to deploy the control plane services. Control flow now depends on
     # the scenario being exercised.
 
+    # Deploy ceph-ansible on ceph-ansible scenarios
+    - block:
+        - name: Run deploy-ceph-ansible.sh script
+          script:
+            cmd: deploy-ceph-ansible.sh
+            executable: /bin/bash
+            chdir: "{{ ceph_ansible_src_dir }}"
+
+        - name: Ensure required kolla config directories exist
+          file:
+            state: directory
+            name: "/etc/kolla/config/{{ item.name }}"
+            mode: 0777
+          with_items: "{{ ceph_ansible_services }}"
+
+        - name: copy ceph.conf to enabled services
+          copy:
+            src: "/etc/ceph/ceph.conf"
+            dest: "/etc/kolla/config/{{ item.name }}/ceph.conf"
+            remote_src: True
+          with_items: "{{ ceph_ansible_services }}"
+
+        - name: copy keyrings to enabled services
+          copy:
+            remote_src: True
+            src: "/etc/ceph/{{ item.keyring }}"
+            dest: "/etc/kolla/config/{{ item.name }}/{{ item.keyring }}"
+          with_items: "{{ ceph_ansible_services }}"
+          become: True
+
+      vars:
+        ceph_ansible_services:
+          - { name: 'cinder/cinder-volume', keyring: "ceph.client.cinder.keyring" }
+          - { name: 'cinder/cinder-backup', keyring: "ceph.client.cinder.keyring" }
+          - { name: 'cinder/cinder-backup', keyring: "ceph.client.cinder-backup.keyring" }
+          - { name: 'glance', keyring: "ceph.client.glance.keyring" }
+          - { name: 'nova', keyring: "ceph.client.nova.keyring" }
+          - { name: 'nova', keyring: "ceph.client.cinder.keyring" }
+      when: scenario == "ceph-ansible"
+
     # Deploy control plane. For upgrade jobs this is the previous release.
     - block:
         - name: Run deploy.sh script
diff --git a/tests/setup_disks.sh b/tests/setup_disks.sh
index 4f3fd9168..bf0ab419b 100644
--- a/tests/setup_disks.sh
+++ b/tests/setup_disks.sh
@@ -54,6 +54,13 @@ elif [ $1 = 'bluestore' ]; then
     LOOP=$(losetup -f)
     losetup $LOOP /opt/data/kolla/ceph-osd0-d.img
     parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_OSD0_D 1 -1
+elif [ $1 = 'ceph-lvm' ]; then
+    free_device=$(losetup -f)
+    fallocate -l 10G /var/lib/ceph-osd1.img
+    losetup $free_device /var/lib/ceph-osd1.img
+    pvcreate $free_device
+    vgcreate cephvg $free_device
+    lvcreate -l 100%FREE -n cephlv cephvg
 else
     echo "Unknown type" >&2
     exit 1
diff --git a/tests/templates/ceph-ansible.j2 b/tests/templates/ceph-ansible.j2
new file mode 100644
index 000000000..17aaf7f79
--- /dev/null
+++ b/tests/templates/ceph-ansible.j2
@@ -0,0 +1,33 @@
+# ceph-ansible group vars
+ceph_stable_release: "nautilus"
+monitor_interface: "{{ api_interface_name }}"
+radosgw_interface: "{{ api_interface_name }}"
+
+public_network: "{{ api_network_prefix }}0/{{ api_network_prefix_length }}"
+
+configure_firewall: false
+
+docker: true
+containerized_deployment: true
+
+dashboard_enabled: false
+
+openstack_config: true
+{% raw %}
+openstack_pools:
+  - "{{ openstack_glance_pool }}"
+  - "{{ openstack_cinder_pool }}"
+  - "{{ openstack_cinder_backup_pool }}"
+  - "{{ openstack_nova_pool }}"
+
+openstack_keys:
+  - { name: client.glance, caps: { mon: "profile rbd", osd: "profile rbd pool=volumes, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" }
+  - { name: client.cinder, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_nova_pool.name }}, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" }
+  - { name: client.cinder-backup, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_backup_pool.name }}"}, mode: "0600" }
+  - { name: client.nova, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_nova_pool.name }}, profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" }
+{% endraw %}
+
+# osds
+lvm_volumes:
+  - data: cephlv
+    data_vg: cephvg
diff --git a/tests/templates/ceph-inventory.j2 b/tests/templates/ceph-inventory.j2
new file mode 100644
index 000000000..17edcdbdc
--- /dev/null
+++ b/tests/templates/ceph-inventory.j2
@@ -0,0 +1,14 @@
+[storage]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }} ceph_osd_store_type={{ 'filestore' if host == 'primary' else 'bluestore' }}
+{% endfor %}
+
+# Ceph-Ansible hosts
+[mons:children]
+storage
+
+[mgrs:children]
+storage
+
+[osds:children]
+storage
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index 2966bf4e5..3db840af9 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -127,3 +127,12 @@ enable_mariadb: "yes"
 enable_memcached: "no"
 enable_rabbitmq: "no"
 {% endif %}
+
+{% if scenario == "ceph-ansible" %}
+# kolla-ansible vars
+enable_cinder: "yes"
+# External Ceph
+glance_backend_ceph: "yes"
+cinder_backend_ceph: "yes"
+nova_backend_ceph: "yes"
+{% endif %}
diff --git a/tests/test-core-openstack.sh b/tests/test-core-openstack.sh
index 351c9048a..267b45f25 100755
--- a/tests/test-core-openstack.sh
+++ b/tests/test-core-openstack.sh
@@ -11,7 +11,7 @@ function test_smoke {
     openstack --debug compute service list
     openstack --debug network agent list
     openstack --debug orchestration service list
-    if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "cinder-lvm" ]]; then
+    if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "ceph-ansible" ]] | [[ $SCENARIO == "cinder-lvm" ]]; then
         openstack --debug volume service list
     fi
 }
@@ -28,7 +28,7 @@ function test_instance_boot {
     fi
     echo "SUCCESS: Server creation"
 
-    if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "cinder-lvm" ]]; then
+    if [[ $SCENARIO == "ceph" ]] || [ $SCENARIO == "ceph-ansible" ] || [[ $SCENARIO == "cinder-lvm" ]]; then
         echo "TESTING: Cinder volume attachment"
         openstack volume create --size 2 test_volume
         attempt=1
diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh
index 46b41cda6..813836c77 100755
--- a/tools/setup_gate.sh
+++ b/tools/setup_gate.sh
@@ -39,6 +39,10 @@ function setup_config {
         GATE_IMAGES+=",ceph,cinder"
     fi
 
+    if [[ $SCENARIO == "ceph-ansible" ]]; then
+        GATE_IMAGES+=",cinder"
+    fi
+
     if [[ $SCENARIO == "cinder-lvm" ]]; then
         GATE_IMAGES+=",cinder,iscsid,tgtd"
     fi
diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml
index d1f338cb0..f053bc7d5 100644
--- a/zuul.d/base.yaml
+++ b/zuul.d/base.yaml
@@ -93,3 +93,14 @@
       - ^tests/test-swift.sh
     vars:
       scenario: swift
+
+- job:
+    name: kolla-ansible-ceph-ansible-base
+    parent: kolla-ansible-base
+    voting: false
+    vars:
+      scenario: ceph-ansible
+      ceph_osd_storetype: ceph-lvm
+    required-projects:
+      - name: github.com/ceph/ceph-ansible
+        override-checkout: v4.0.7
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index d5505d6e3..d1f26e2a5 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -96,6 +96,24 @@
         ceph_osd_storetype: bluestore
 
 - job:
+    name: kolla-ansible-centos-source-ceph-ansible
+    parent: kolla-ansible-ceph-ansible-base
+    nodeset: kolla-ansible-centos-multi
+    timeout: 9000
+    vars:
+      base_distro: centos
+      install_type: source
+
+- job:
+    name: kolla-ansible-ubuntu-source-ceph-ansible
+    parent: kolla-ansible-ceph-ansible-base
+    nodeset: kolla-ansible-bionic-multi
+    timeout: 9000
+    vars:
+      base_distro: ubuntu
+      install_type: source
+
+- job:
     name: kolla-ansible-ubuntu-source-cinder-lvm
     parent: kolla-ansible-base
     nodeset: kolla-ansible-bionic-multi
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 35090c511..3c7f16636 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -70,6 +70,14 @@
             files:
               - ^ansible/roles/mariadb/
               - ^tests/test-mariadb.sh
+        - kolla-ansible-centos-source-ceph-ansible:
+            files:
+              - ^ansible/roles/(cinder|glance|gnocchi|nova-cell)/
+              - ^tests/deploy-ceph-ansible.sh
+        - kolla-ansible-ubuntu-source-ceph-ansible:
+            files:
+              - ^ansible/roles/(cinder|glance|gnocchi|nova-cell)/
+              - ^tests/deploy-ceph-ansible.sh
     check-arm64:
       jobs:
         - kolla-ansible-debian-source-aarch64