Playbook for nova upgrade

This creates tree and playbook for nova upgrade. Also other service
upgrades will follow standard setup here.

Change-Id: Ic31759efaee4986eb87b9ff0968f13189d130d48
Partially-Implements: blueprint upgrade-kolla
Implements: blueprint upgrade-nova

9 files changed, 287 insertions, 189 deletions

diff --git a/ansible/library/kolla_docker.py b/ansible/library/kolla_docker.py
index f095c4db8..3b1e9bd61 100644
--- a/ansible/library/kolla_docker.py
+++ b/ansible/library/kolla_docker.py
@@ -33,11 +33,13 @@ options:
     required: True
     type: str
     choices:
+      - compare_image
       - create_volume
       - pull_image
       - remove_container
       - remove_volume
       - start_container
+      - stop_container
   api_version:
     description:
       - The version of the api for docker-py to use when contacting docker
@@ -230,12 +232,14 @@ class DockerWorker(object):
             if find_name in cont['Names']:
                 return cont
 
-    def check_container_differs(self):
+    def get_container_info(self):
         container = self.check_container()
         if not container:
-            return True
-        container_info = self.dc.inspect_container(self.params.get('name'))
+            return None
+        return self.dc.inspect_container(self.params.get('name'))
 
+    def check_container_differs(self):
+        container_info = self.get_container_info()
         return (
             self.compare_image(container_info) or
             self.compare_labels(container_info) or
@@ -261,7 +265,10 @@ class DockerWorker(object):
         if new_privileged != current_privileged:
             return True
 
-    def compare_image(self, container_info):
+    def compare_image(self, container_info=None):
+        container_info = container_info or self.get_container_info()
+        if not container_info:
+            return True
         new_image = self.check_image()
         current_image = container_info['Image']
         if new_image['Id'] != current_image:
@@ -493,6 +500,13 @@ class DockerWorker(object):
             if self.params.get('remove_on_exit'):
                 self.remove_container()
 
+    def stop_container(self):
+        name = self.params.get('name')
+        container = self.check_container()
+        if not container['Status'].startswith('Exited '):
+            self.changed = True
+            self.dc.stop(name)
+
     def create_volume(self):
         if not self.check_volume():
             self.changed = True
@@ -517,11 +531,13 @@ class DockerWorker(object):
 def generate_module():
     argument_spec = dict(
         common_options=dict(required=False, type='dict', default=dict()),
-        action=dict(requried=True, type='str', choices=['create_volume',
+        action=dict(requried=True, type='str', choices=['compare_image',
+                                                        'create_volume',
                                                         'pull_image',
                                                         'remove_container',
                                                         'remove_volume',
-                                                        'start_container']),
+                                                        'start_container',
+                                                        'stop_container']),
         api_version=dict(required=False, type='str', default='auto'),
         auth_email=dict(required=False, type='str'),
         auth_password=dict(required=False, type='str'),
@@ -594,8 +610,11 @@ def main():
 
     try:
         dw = DockerWorker(module)
-        getattr(dw, module.params.get('action'))()
-        module.exit_json(changed=dw.changed)
+        # TODO(inc0): We keep it bool to have ansible deal with cosistent
+        # types. If we ever add method that will have to return some
+        # meaningful data, we need to refactor all methods to return dicts.
+        result = bool(getattr(dw, module.params.get('action'))())
+        module.exit_json(changed=dw.changed, result=result)
     except Exception as e:
         module.exit_json(failed=True, changed=True, msg=repr(e))
 
diff --git a/ansible/roles/nova/tasks/bootstrap.yml b/ansible/roles/nova/tasks/bootstrap.yml
index f67d44091..e7014527c 100644
--- a/ansible/roles/nova/tasks/bootstrap.yml
+++ b/ansible/roles/nova/tasks/bootstrap.yml
@@ -71,51 +71,5 @@
   run_once: True
   delegate_to: "{{ groups['nova-api'][0] }}"
 
-- name: Running Nova bootstrap container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    detach: False
-    environment:
-      KOLLA_BOOTSTRAP:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-    image: "{{ nova_api_image_full }}"
-    labels:
-      BOOTSTRAP:
-    name: "bootstrap_nova"
-    restart_policy: "never"
-    volumes: "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
-  run_once: True
-  delegate_to: "{{ groups['nova-api'][0] }}"
+- include: bootstrap_service.yml
   when: database_created
-
-- name: Creating nova-compute volume
-  kolla_docker:
-    action: "create_volume"
-    common_options: "{{ docker_common_options }}"
-    name: "nova_compute"
-  register: nova_compute_volume
-  when:
-    - inventory_hostname in groups['compute']
-    - not enable_nova_fake | bool
-
-- name: Running Nova compute bootstrap container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    detach: False
-    environment:
-      KOLLA_BOOTSTRAP:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-    image: "{{ nova_compute_image_full }}"
-    labels:
-      BOOTSTRAP:
-    name: "bootstrap_nova_compute"
-    restart_policy: "never"
-    volumes:
-      - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
-      - "nova_compute:/var/lib/nova/"
-  when:
-    - inventory_hostname in groups['compute']
-    - not enable_nova_fake | bool
-    - nova_compute_volume.changed
diff --git a/ansible/roles/nova/tasks/bootstrap_service.yml b/ansible/roles/nova/tasks/bootstrap_service.yml
new file mode 100644
index 000000000..787b2787b
--- /dev/null
+++ b/ansible/roles/nova/tasks/bootstrap_service.yml
@@ -0,0 +1,48 @@
+---
+- name: Running Nova bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ nova_api_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_nova"
+    restart_policy: "never"
+    volumes: "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
+  run_once: True
+  delegate_to: "{{ groups['nova-api'][0] }}"
+
+- name: Creating nova-compute volume
+  kolla_docker:
+    action: "create_volume"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_compute"
+  register: nova_compute_volume
+  when:
+    - inventory_hostname in groups['compute']
+    - not enable_nova_fake | bool
+
+- name: Running Nova compute bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ nova_compute_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_nova_compute"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
+      - "nova_compute:/var/lib/nova/"
+  when:
+    - inventory_hostname in groups['compute']
+    - not enable_nova_fake | bool
+    - nova_compute_volume.changed
diff --git a/ansible/roles/nova/tasks/reload.yml b/ansible/roles/nova/tasks/reload.yml
new file mode 100644
index 000000000..25c959e14
--- /dev/null
+++ b/ansible/roles/nova/tasks/reload.yml
@@ -0,0 +1,33 @@
+---
+# This play calls sighup on every service to refresh upgrade levels
+- name: Sighup nova-api
+  command: docker exec -t nova_api kill -1 1
+  when: inventory_hostname in groups['nova-api']
+
+- name: Sighup nova-conductor
+  command: docker exec -t nova_conductor kill -1 1
+  when: inventory_hostname in groups['nova-conductor']
+
+- name: Sighup nova-consoleauth
+  command: docker exec -t nova_consoleauth kill -1 1
+  when: inventory_hostname in groups['nova-consoleauth']
+
+- name: Sighup nova-novncproxy
+  command: docker exec -t nova_novncproxy kill -1 1
+  when:
+    - inventory_hostname in groups['nova-novncproxy']
+    - nova_console == 'novnc'
+
+- name: Sighup nova-scheduler
+  command: docker exec -t nova_scheduler kill -1 1
+  when: inventory_hostname in groups['nova-scheduler']
+
+- name: Sighup nova-spicehtml5proxy
+  command: docker exec -t nova_spicehtml5proxy kill -1 1
+  when:
+    - inventory_hostname in groups['nova-spicehtml5proxy']
+    - nova_console == 'spice'
+
+- name: Sighup nova-compute
+  command: docker exec -t nova_compute kill -1 1
+  when: inventory_hostname in groups['compute']
diff --git a/ansible/roles/nova/tasks/start.yml b/ansible/roles/nova/tasks/start.yml
index 795e2ca5d..659729b19 100644
--- a/ansible/roles/nova/tasks/start.yml
+++ b/ansible/roles/nova/tasks/start.yml
@@ -1,137 +1,6 @@
 ---
-- name: Starting nova-libvirt container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_libvirt_image_full }}"
-    name: "nova_libvirt"
-    pid_mode: "host"
-    privileged: True
-    volumes:
-      - "{{ node_config_directory }}/nova-libvirt/:{{ container_config_directory }}/:ro"
-      - "/lib/modules:/lib/modules:ro"
-      - "/run/:/run/"
-      - "/sys/fs/cgroup:/sys/fs/cgroup"
-      - "nova_compute:/var/lib/nova/"
-      - "nova_libvirt:/var/lib/libvirt"
-  when: inventory_hostname in groups['compute']
+- include: start_controllers.yml
 
-- name: Starting nova-api container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_api_image_full }}"
-    name: "nova_api"
-    privileged: True
-    volumes:
-      - "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
-      - "/lib/modules:/lib/modules:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when: inventory_hostname in groups['nova-api']
+- include: start_conductors.yml
 
-- name: Starting nova-conductor container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    name: "nova_conductor"
-    image: "{{ nova_conductor_image_full }}"
-    volumes:
-      - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when: inventory_hostname in groups['nova-conductor']
-
-- name: Starting nova-consoleauth container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_consoleauth_image_full }}"
-    name: "nova_consoleauth"
-    volumes:
-      - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when: inventory_hostname in groups['nova-consoleauth']
-
-- name: Starting nova-novncproxy container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_novncproxy_image_full }}"
-    name: "nova_novncproxy"
-    volumes:
-      - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when:
-    - inventory_hostname in groups['nova-novncproxy']
-    - nova_console == 'novnc'
-
-- name: Starting nova-scheduler container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_scheduler_image_full }}"
-    name: "nova_scheduler"
-    volumes:
-      - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when: inventory_hostname in groups['nova-scheduler']
-
-- name: Starting nova-spicehtml5proxy container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_spicehtml5proxy_image_full }}"
-    name: "nova_spicehtml5proxy"
-    volumes:
-      - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when:
-    - inventory_hostname in groups['nova-spicehtml5proxy']
-    - nova_console == 'spice'
-
-- name: Starting nova-compute container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_compute_image_full }}"
-    name: "nova_compute"
-    privileged: True
-    volumes:
-      - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
-      - "/lib/modules:/lib/modules:ro"
-      - "/run:/run"
-      - "nova_compute:/var/lib/nova/"
-      - "nova_libvirt:/var/lib/libvirt"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when:
-    - inventory_hostname in groups['compute']
-    - not enable_nova_fake | bool
-
-- name: Starting nova-compute-ironic container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_compute_ironic_image_full }}"
-    name: "nova_compute_ironic"
-    volumes:
-      - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  when:
-    - inventory_hostname in groups['nova-compute-ironic']
-    - enable_ironic | bool
-
-- name: Starting nova-compute-fake containers
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ nova_compute_image_full }}"
-    name: "nova_compute_fake_{{ item }}"
-    privileged: True
-    volumes:
-      - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro"
-      - "/lib/modules:/lib/modules:ro"
-      - "/run:/run"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
-  with_sequence: start=1 end={{ num_nova_fake_per_node }}
-  when:
-    - inventory_hostname in groups['compute']
-    - enable_nova_fake | bool
+- include: start_compute.yml
diff --git a/ansible/roles/nova/tasks/start_compute.yml b/ansible/roles/nova/tasks/start_compute.yml
new file mode 100644
index 000000000..bbb8419cb
--- /dev/null
+++ b/ansible/roles/nova/tasks/start_compute.yml
@@ -0,0 +1,65 @@
+---
+- name: Starting nova-libvirt container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_libvirt_image_full }}"
+    name: "nova_libvirt"
+    pid_mode: "host"
+    privileged: True
+    volumes:
+      - "{{ node_config_directory }}/nova-libvirt/:{{ container_config_directory }}/:ro"
+      - "/lib/modules:/lib/modules:ro"
+      - "/run/:/run/"
+      - "/sys/fs/cgroup:/sys/fs/cgroup"
+      - "nova_compute:/var/lib/nova/"
+      - "nova_libvirt:/var/lib/libvirt"
+  when: inventory_hostname in groups['compute']
+
+- name: Starting nova-compute container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_compute_image_full }}"
+    name: "nova_compute"
+    privileged: True
+    volumes:
+      - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
+      - "/lib/modules:/lib/modules:ro"
+      - "/run:/run"
+      - "nova_compute:/var/lib/nova/"
+      - "nova_libvirt:/var/lib/libvirt"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when:
+    - inventory_hostname in groups['compute']
+    - not enable_nova_fake | bool
+
+- name: Starting nova-compute-ironic container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_compute_ironic_image_full }}"
+    name: "nova_compute_ironic"
+    volumes:
+      - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when:
+    - inventory_hostname in groups['nova-compute-ironic']
+    - enable_ironic | bool
+
+- name: Starting nova-compute-fake containers
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_compute_image_full }}"
+    name: "nova_compute_fake_{{ item }}"
+    privileged: True
+    volumes:
+      - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro"
+      - "/lib/modules:/lib/modules:ro"
+      - "/run:/run"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  with_sequence: start=1 end={{ num_nova_fake_per_node }}
+  when:
+    - inventory_hostname in groups['compute']
+    - enable_nova_fake | bool
diff --git a/ansible/roles/nova/tasks/start_conductors.yml b/ansible/roles/nova/tasks/start_conductors.yml
new file mode 100644
index 000000000..11ebf2ff3
--- /dev/null
+++ b/ansible/roles/nova/tasks/start_conductors.yml
@@ -0,0 +1,11 @@
+---
+- name: Starting nova-conductor container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+    image: "{{ nova_conductor_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when: inventory_hostname in groups['nova-conductor']
diff --git a/ansible/roles/nova/tasks/start_controllers.yml b/ansible/roles/nova/tasks/start_controllers.yml
new file mode 100644
index 000000000..c9b666800
--- /dev/null
+++ b/ansible/roles/nova/tasks/start_controllers.yml
@@ -0,0 +1,61 @@
+---
+- name: Starting nova-api container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_api_image_full }}"
+    name: "nova_api"
+    privileged: True
+    volumes:
+      - "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
+      - "/lib/modules:/lib/modules:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when: inventory_hostname in groups['nova-api']
+
+- name: Starting nova-consoleauth container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_consoleauth_image_full }}"
+    name: "nova_consoleauth"
+    volumes:
+      - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when: inventory_hostname in groups['nova-consoleauth']
+
+- name: Starting nova-novncproxy container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_novncproxy_image_full }}"
+    name: "nova_novncproxy"
+    volumes:
+      - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when:
+    - inventory_hostname in groups['nova-novncproxy']
+    - nova_console == 'novnc'
+
+- name: Starting nova-scheduler container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_scheduler_image_full }}"
+    name: "nova_scheduler"
+    volumes:
+      - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when: inventory_hostname in groups['nova-scheduler']
+
+- name: Starting nova-spicehtml5proxy container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ nova_spicehtml5proxy_image_full }}"
+    name: "nova_spicehtml5proxy"
+    volumes:
+      - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro"
+      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+  when:
+    - inventory_hostname in groups['nova-spicehtml5proxy']
+    - nova_console == 'spice'
diff --git a/ansible/roles/nova/tasks/upgrade.yml b/ansible/roles/nova/tasks/upgrade.yml
index ed97d539c..f8f2f7dee 100644
--- a/ansible/roles/nova/tasks/upgrade.yml
+++ b/ansible/roles/nova/tasks/upgrade.yml
@@ -1 +1,39 @@
 ---
+# Create new set of configs on nodes
+- include: config.yml
+
+# TODO(inc0): since nova is creating new database in L->M, we need to call it.
+# It should be removed later
+- include: bootstrap.yml
+
+- include: bootstrap_service.yml
+
+- name: Checking if conductor container needs upgrading
+  kolla_docker:
+    action: "compare_image"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+    image: "{{ nova_conductor_image_full }}"
+  when: inventory_hostname in groups['nova-conductor']
+  register: conductor_differs
+
+# Short downtime here, but from user perspective his call will just timeout or execute later
+- name: Stopping all nova_conductor containers
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+  when:
+    - inventory_hostname in groups['nova-conductor']
+    - conductor_differs['result']
+
+- include: start_conductors.yml
+
+- include: start_controllers.yml
+  serial: "30%"
+
+- include: start_compute.yml
+  serial: "10%"
+
+- include: reload.yml
+  serial: "30%"