libvirt: add nova-libvirt-cleanup command

Change Ia1239069ccee39416b20959cbabad962c56693cf added support for
running a libvirt daemon on the host, rather than using the nova_libvirt
container. It did not cover migration of existing hosts from using a
container to using a host daemon.

This change adds a kolla-ansible nova-libvirt-cleanup command which may
be used to clean up the nova_libvirt container, volumes and related
items on hosts, once it has been disabled.

The playbook assumes that compute hosts have been emptied of VMs before
it runs. A future extension could support migration of existing VMs, but
this is currently out of scope.

Change-Id: I46854ed7eaf1d5b5e3ccd8531c963427848bdc99

6 files changed, 138 insertions, 2 deletions

diff --git a/ansible/nova-libvirt-cleanup.yml b/ansible/nova-libvirt-cleanup.yml
new file mode 100644
index 000000000..5124b37d4
--- /dev/null
+++ b/ansible/nova-libvirt-cleanup.yml
@@ -0,0 +1,14 @@
+---
+- import_playbook: gather-facts.yml
+
+- name: Remove nova_libvirt container
+  gather_facts: false
+  hosts:
+    - compute
+  serial: '{{ kolla_serial|default("0") }}'
+  tags:
+    - nova-libvirt-cleanup
+  tasks:
+    - import_role:
+        name: nova-cell
+        tasks_from: libvirt-cleanup.yml
diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml
index 9f370f537..2247cece8 100644
--- a/ansible/roles/nova-cell/defaults/main.yml
+++ b/ansible/roles/nova-cell/defaults/main.yml
@@ -558,3 +558,14 @@ enable_shared_var_lib_nova_mnt: "{{ enable_cinder_backend_nfs | bool or enable_c
 ###################################
 
 nova_pci_passthrough_whitelist: "{{ enable_neutron_sriov | bool | ternary(neutron_sriov_physnet_mappings | dict2items(key_name='physical_network', value_name='devname'), []) }}"
+
+##################
+# Libvirt cleanup
+##################
+
+# The following options pertain to the kolla-ansible nova-libvirt-cleanup command.
+
+# Whether to fail when there are running VMs.
+nova_libvirt_cleanup_running_vms_fatal: true
+# Whether to remove Docker volumes.
+nova_libvirt_cleanup_remove_volumes: false
diff --git a/ansible/roles/nova-cell/tasks/libvirt-cleanup.yml b/ansible/roles/nova-cell/tasks/libvirt-cleanup.yml
new file mode 100644
index 000000000..4ec206f44
--- /dev/null
+++ b/ansible/roles/nova-cell/tasks/libvirt-cleanup.yml
@@ -0,0 +1,80 @@
+---
+- name: Fail if nova_libvirt container is enabled
+  fail:
+    msg: >-
+      The nova_libvirt container has not been cleaned up because it is enabled.
+      It may be disabled by setting enable_nova_libvirt_container to false.
+  when: enable_nova_libvirt_container | bool
+
+- name: Get container facts
+  become: true
+  kolla_container_facts:
+    name:
+      - nova_libvirt
+  register: container_facts
+
+- block:
+    - name: Check if there are any running VMs
+      become: true
+      shell:
+        cmd: >
+          set -o pipefail &&
+          pgrep -l qemu | awk '!/qemu-ga/  && !/qemu-img/ {print $1}'
+      register: running_vms
+
+    - name: Fail if there are any running VMs
+      fail:
+        msg: >-
+          Refusing to remove nova_libvirt container with running VMs:
+          {{ running_vms.stdout }}
+      when:
+        - running_vms.stdout != ''
+        - nova_libvirt_cleanup_running_vms_fatal | bool
+
+    - name: Stop and remove nova_libvirt container
+      become: true
+      kolla_docker:
+        action: "stop_and_remove_container"
+        name: nova_libvirt
+  when: container_facts['nova_libvirt'] is defined
+
+- name: Remove nova_libvirt Docker volumes
+  become: true
+  kolla_docker:
+    action: "remove_volume"
+    name: "{{ item }}"
+  loop:
+    - libvirtd
+    - nova_libvirt_qemu
+    - nova_libvirt_secrets
+  when: nova_libvirt_cleanup_remove_volumes | bool
+
+- name: Remove config for nova_libvirt
+  become: true
+  file:
+    path: "{{ node_config_directory }}/nova-libvirt"
+    state: "absent"
+
+# Revert the changes applied in config-host.yml.
+- block:
+    - name: Remove udev kolla kvm rules
+      become: true
+      file:
+        path: "/etc/udev/rules.d/99-kolla-kvm.rules"
+        state: absent
+
+    - name: Reset /dev/kvm ownership
+      become: true
+      file:
+        path: /dev/kvm
+        group: kvm
+
+    - name: Unmask qemu-kvm service
+      become: true
+      systemd:
+        name: qemu-kvm.service
+        masked: false
+      when:
+        - ansible_facts.distribution == 'Ubuntu'
+  when:
+    - nova_compute_virt_type == 'kvm'
diff --git a/doc/source/reference/compute/libvirt-guide.rst b/doc/source/reference/compute/libvirt-guide.rst
index 6a33a0177..13ce8e4b2 100644
--- a/doc/source/reference/compute/libvirt-guide.rst
+++ b/doc/source/reference/compute/libvirt-guide.rst
@@ -54,8 +54,27 @@ libvirt as a host daemon. However, since the Yoga release, if a libvirt daemon
 has already been set up, then Kolla Ansible may be configured to use it. This
 may be achieved by setting ``enable_nova_libvirt_container`` to ``false``.
 
-Migration of hosts from a containerised libvirt to host libvirt is currently
-not supported.
+Migration from container to host
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``kolla-ansible nova-libvirt-cleanup`` command may be used to clean up the
+``nova_libvirt`` container and related items on hosts, once it has
+been disabled. This should be run after the compute service has been disabled,
+and all active VMs have been migrated away from the host.
+
+By default, the command will fail if there are any VMs running on the host. If
+you are sure that it is safe to clean up the ``nova_libvirt`` container with
+running VMs, setting ``nova_libvirt_cleanup_running_vms_fatal`` to ``false``
+will allow the command to proceed.
+
+The ``nova_libvirt`` container has several associated Docker volumes:
+``libvirtd``, ``nova_libvirt_qemu`` and ``nova_libvirt_secrets``. By default,
+these volumes are not cleaned up. If you are sure that the data in these
+volumes can be safely removed, setting ``nova_libvirt_cleanup_remove_volumes``
+to ``true`` will cause the Docker volumes to be removed.
+
+A future extension could support migration of existing VMs, but this is
+currently out of scope.
 
 .. libvirt-tls:
 
diff --git a/releasenotes/notes/nova-libvirt-cleanup-0d2c3b3156f2f657.yaml b/releasenotes/notes/nova-libvirt-cleanup-0d2c3b3156f2f657.yaml
new file mode 100644
index 000000000..25f5833e4
--- /dev/null
+++ b/releasenotes/notes/nova-libvirt-cleanup-0d2c3b3156f2f657.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds a ``kolla-ansible nova-libvirt-cleanup`` command, which may be used to
+    clean up the ``nova_libvirt`` container. This may be useful if switching to
+    a host libvirt daemon.
diff --git a/tools/kolla-ansible b/tools/kolla-ansible
index 23c08f23b..2f80ff67a 100755
--- a/tools/kolla-ansible
+++ b/tools/kolla-ansible
@@ -198,6 +198,7 @@ Commands:
     upgrade-bifrost      Upgrades an existing bifrost container
     genconfig            Generate configuration files for enabled OpenStack services
     prune-images         Prune orphaned Kolla images
+    nova-libvirt-cleanup Clean up disabled nova_libvirt containers
 EOF
 }
 
@@ -244,6 +245,7 @@ upgrade
 upgrade-bifrost
 genconfig
 prune-images
+nova-libvirt-cleanup
 EOF
 }
 
@@ -547,6 +549,10 @@ EOF
             exit 1
         fi
         ;;
+(nova-libvirt-cleanup)
+        ACTION="Cleanup disabled nova_libvirt containers"
+        PLAYBOOK="${BASEDIR}/ansible/nova-libvirt-cleanup.yml"
+        ;;
 (bash-completion)
         bash_completion
         exit 0