Clean up test process

If stestr failed, then no node info was logged.
Logging node info is now handled in pre and post stages of Zuul.
Additionally Zuul v3 migration recommendations were applied
and some unused code removed improving general readability.

Change-Id: Iae9d37cb76997ba8389e21c7e2a3bf5905abe029
Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
(cherry picked from commit c7ac0f0de0c65afe54974a7ea727feb27c0f60eb)

5 files changed, 58 insertions, 85 deletions

diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml
index bb514821d..00f71f6b9 100644
--- a/tests/playbooks/post.yml
+++ b/tests/playbooks/post.yml
@@ -1,60 +1,66 @@
 ---
 - hosts: all
   tasks:
-    - shell:
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Collect various info to logs
+      shell:
         cmd: |
           set +o errexit
-          mkdir -p logs
-          # copy system log
+          set -o xtrace
 
-          sudo cp -r /var/log logs/system_log
+          # copy system log
+          sudo cp -r /var/log system_log
 
           if which journalctl ; then
               # the journal gives us syslog() and kernel output, so is like
               # a concatenation of the above.
-              sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
-              sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
+              sudo journalctl --no-pager > syslog.txt
+              sudo journalctl --no-pager -u docker.service > docker.log
           else
               # assume rsyslog
-              sudo cp /var/log/syslog logs/syslog.txt
-              sudo cp /var/log/kern.log logs/kern_log.txt
-              sudo cp /var/log/upstart/docker.log logs/docker.log
+              sudo cp /var/log/syslog syslog.txt
+              sudo cp /var/log/kern.log kern_log.txt
+              sudo cp /var/log/upstart/docker.log docker.log
           fi
 
           if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
-              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
+              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs
           fi
 
           # sudo config
-          sudo cp -r /etc/sudoers.d logs/
-          sudo cp /etc/sudoers logs/sudoers.txt
+          sudo cp -r /etc/sudoers.d ./
+          sudo cp /etc/sudoers sudoers.txt
 
-          df -h > logs/df.txt
-          free  > logs/free.txt
-          sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
-          mount > logs/mount.txt
-          env > logs/env.txt
+          df -h > df.txt
+          free  > free.txt
+          sudo parted -l > parted-l.txt
+          mount > mount.txt
+          env > env.txt
 
           if [ `command -v dpkg` ]; then
-              dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
+              dpkg -l > dpkg-l.txt
           fi
           if [ `command -v rpm` ]; then
-              rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
+              rpm -qa > rpm-qa.txt
           fi
 
           # final memory usage and process list
-          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
+          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt
 
           # docker related information
-          (docker info && docker system df && docker system df -v) > logs/docker-info.txt
+          (docker info && docker system df && docker system df -v) > docker-info.txt
 
-          sudo cp -r /etc/kolla logs/kolla_configs
+          sudo cp -r /etc/kolla kolla_configs
 
-          # fix the permissions for logs folder
-          sudo chmod -R 777 logs
+          # last sudo to own all files
+          sudo chown -R {{ ansible_user_id }}: .
 
           # Parse build logs and extract pkg install info
-          tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt
+          {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt
 
           # rename files to .txt; this is so that when displayed via
           # logs.openstack.org clicking results in the browser shows the
@@ -62,30 +68,21 @@
           # download it, etc.
 
           # firstly, rename all .log files to .txt files
-          for f in $(find logs -name "*.log"); do
-              sudo mv $f ${f/.log/.txt}
+          for f in $(find . -name "*.log"); do
+              mv $f ${f/.log/.txt}
           done
 
           # Update symlinks to new file names
-          for f in $(find logs -name "*FAILED*"); do
-              sudo mv ${f} ${f}.gz
-              sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
+          for f in $(find . -name "*FAILED*"); do
+              mv ${f} ${f}.gz
+              ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
           done
 
           # append .txt to all kolla config file
-          find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
+          find kolla_configs -type f -exec mv '{}' '{}'.txt \;
 
           # Compress all text logs
-          find logs -iname '*.txt' -execdir gzip -9 {} \+
-          find logs -iname '*.json' -execdir gzip -9 {} \+
+          find . -iname '*.txt' -execdir gzip -9 {} \+
+          find . -iname '*.json' -execdir gzip -9 {} \+
         executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
-    - synchronize:
-        src: '{{ zuul.project.src_dir }}/logs'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - "--quiet"
+        chdir: "{{ kolla_logs_dir }}"
diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml
index 00d471614..e721d8859 100644
--- a/tests/playbooks/pre.yml
+++ b/tests/playbooks/pre.yml
@@ -1,13 +1,17 @@
 ---
 - hosts: all
-  become: true
   tasks:
-    - name: Install dbus for debian system
-      apt: name=dbus
-      when: ansible_os_family == 'Debian'
+    - name: Create dir for kolla logs
+      file:
+        path: "{{ kolla_logs_dir }}"
+        state: directory
 
-    - name: Install ansible
-      pip:
-        name: ansible
-      when:
-        - inventory_hostname == "primary"
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Create dir for kolla build logs
+      file:
+        path: "{{ kolla_build_logs_dir }}"
+        state: directory
diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml
index b15ec45b9..0578dea32 100644
--- a/tests/playbooks/run.yml
+++ b/tests/playbooks/run.yml
@@ -1,7 +1,6 @@
 ---
 - hosts: all
   vars:
-    kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory"
     nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo"
     nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple"
     nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}"
@@ -12,23 +11,6 @@
     nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/"
     nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona"
   tasks:
-    - name: generate kolla inventory file
-      copy:
-        dest: "{{ kolla_inventory_path }}"
-        content: |
-          {% for host in hostvars %}
-          {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }}
-          {% endfor %}
-
-    - name: setup logging
-      shell:
-        cmd: |
-          mkdir logs
-          ln -s $(pwd)/logs /tmp/logs
-          mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs}
-        executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
     - name: copy setup script
       copy:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh"
@@ -57,21 +39,12 @@
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
         dest: /etc/kolla/kolla-build.conf
 
-    - name: Template override
+    - name: Template template_overrides.j2
       template:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2"
         dest: /etc/kolla/template_overrides.j2
 
-    - shell:
-        cmd: |
-          set -e
-          set -x
-
-          export ACTION={{ action }}
-          export BASE_DISTRO={{ base_distro }}
-          export INSTALL_TYPE={{ install_type }}
-          export IN_PROJECT_JOBS=1
-          tools/gate_run.sh
-        executable: /bin/bash
+    - name: Run tox
+      command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }}
+      args:
         chdir: "{{ zuul.project.src_dir }}"
-      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh
deleted file mode 120000
index fbacb3f92..000000000
--- a/tests/setup_gate.sh
+++ /dev/null
@@ -1 +0,0 @@
-../tools/setup_gate.sh
\ No newline at end of file
diff --git a/tests/templates/kolla-build.conf.j2 b/tests/templates/kolla-build.conf.j2
index 3f9e5d04c..37b06aca6 100644
--- a/tests/templates/kolla-build.conf.j2
+++ b/tests/templates/kolla-build.conf.j2
@@ -5,5 +5,5 @@ namespace = kolla
 tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }}
 {% endif %}
 push = false
-logs_dir = /tmp/logs/build
+logs_dir = {{ kolla_build_logs_dir }}
 template_override = /etc/kolla/template_overrides.j2