Clean up test process

If stestr failed, then no node info was logged.
Logging node info is now handled in pre and post stages of Zuul.
Additionally Zuul v3 migration recommendations were applied
and some unused code removed improving general readability.

Change-Id: Iae9d37cb76997ba8389e21c7e2a3bf5905abe029
Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>

10 files changed, 62 insertions, 133 deletions

diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml
index 0e2fbbec5..2fc3dcf08 100644
--- a/.zuul.d/base.yaml
+++ b/.zuul.d/base.yaml
@@ -46,9 +46,6 @@
     run: tests/playbooks/run.yml
     post-run: tests/playbooks/post.yml
     attempts: 1
-    required-projects:
-      - openstack/kolla-ansible
-      - openstack/requirements
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*
@@ -59,3 +56,6 @@
       - ^test-requirements.txt$
     vars:
       publisher: false
+    extra-vars:
+      kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla"
+      kolla_build_logs_dir: "{{ kolla_logs_dir }}/build"
diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml
index bb514821d..00f71f6b9 100644
--- a/tests/playbooks/post.yml
+++ b/tests/playbooks/post.yml
@@ -1,60 +1,66 @@
 ---
 - hosts: all
   tasks:
-    - shell:
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Collect various info to logs
+      shell:
         cmd: |
           set +o errexit
-          mkdir -p logs
-          # copy system log
+          set -o xtrace
 
-          sudo cp -r /var/log logs/system_log
+          # copy system log
+          sudo cp -r /var/log system_log
 
           if which journalctl ; then
               # the journal gives us syslog() and kernel output, so is like
               # a concatenation of the above.
-              sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
-              sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
+              sudo journalctl --no-pager > syslog.txt
+              sudo journalctl --no-pager -u docker.service > docker.log
           else
               # assume rsyslog
-              sudo cp /var/log/syslog logs/syslog.txt
-              sudo cp /var/log/kern.log logs/kern_log.txt
-              sudo cp /var/log/upstart/docker.log logs/docker.log
+              sudo cp /var/log/syslog syslog.txt
+              sudo cp /var/log/kern.log kern_log.txt
+              sudo cp /var/log/upstart/docker.log docker.log
           fi
 
           if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
-              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
+              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs
           fi
 
           # sudo config
-          sudo cp -r /etc/sudoers.d logs/
-          sudo cp /etc/sudoers logs/sudoers.txt
+          sudo cp -r /etc/sudoers.d ./
+          sudo cp /etc/sudoers sudoers.txt
 
-          df -h > logs/df.txt
-          free  > logs/free.txt
-          sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
-          mount > logs/mount.txt
-          env > logs/env.txt
+          df -h > df.txt
+          free  > free.txt
+          sudo parted -l > parted-l.txt
+          mount > mount.txt
+          env > env.txt
 
           if [ `command -v dpkg` ]; then
-              dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
+              dpkg -l > dpkg-l.txt
           fi
           if [ `command -v rpm` ]; then
-              rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
+              rpm -qa > rpm-qa.txt
           fi
 
           # final memory usage and process list
-          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
+          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt
 
           # docker related information
-          (docker info && docker system df && docker system df -v) > logs/docker-info.txt
+          (docker info && docker system df && docker system df -v) > docker-info.txt
 
-          sudo cp -r /etc/kolla logs/kolla_configs
+          sudo cp -r /etc/kolla kolla_configs
 
-          # fix the permissions for logs folder
-          sudo chmod -R 777 logs
+          # last sudo to own all files
+          sudo chown -R {{ ansible_user_id }}: .
 
           # Parse build logs and extract pkg install info
-          tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt
+          {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt
 
           # rename files to .txt; this is so that when displayed via
           # logs.openstack.org clicking results in the browser shows the
@@ -62,30 +68,21 @@
           # download it, etc.
 
           # firstly, rename all .log files to .txt files
-          for f in $(find logs -name "*.log"); do
-              sudo mv $f ${f/.log/.txt}
+          for f in $(find . -name "*.log"); do
+              mv $f ${f/.log/.txt}
           done
 
           # Update symlinks to new file names
-          for f in $(find logs -name "*FAILED*"); do
-              sudo mv ${f} ${f}.gz
-              sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
+          for f in $(find . -name "*FAILED*"); do
+              mv ${f} ${f}.gz
+              ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
           done
 
           # append .txt to all kolla config file
-          find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
+          find kolla_configs -type f -exec mv '{}' '{}'.txt \;
 
           # Compress all text logs
-          find logs -iname '*.txt' -execdir gzip -9 {} \+
-          find logs -iname '*.json' -execdir gzip -9 {} \+
+          find . -iname '*.txt' -execdir gzip -9 {} \+
+          find . -iname '*.json' -execdir gzip -9 {} \+
         executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
-    - synchronize:
-        src: '{{ zuul.project.src_dir }}/logs'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - "--quiet"
+        chdir: "{{ kolla_logs_dir }}"
diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml
index 00d471614..e721d8859 100644
--- a/tests/playbooks/pre.yml
+++ b/tests/playbooks/pre.yml
@@ -1,13 +1,17 @@
 ---
 - hosts: all
-  become: true
   tasks:
-    - name: Install dbus for debian system
-      apt: name=dbus
-      when: ansible_os_family == 'Debian'
+    - name: Create dir for kolla logs
+      file:
+        path: "{{ kolla_logs_dir }}"
+        state: directory
 
-    - name: Install ansible
-      pip:
-        name: ansible
-      when:
-        - inventory_hostname == "primary"
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Create dir for kolla build logs
+      file:
+        path: "{{ kolla_build_logs_dir }}"
+        state: directory
diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml
index b15ec45b9..0578dea32 100644
--- a/tests/playbooks/run.yml
+++ b/tests/playbooks/run.yml
@@ -1,7 +1,6 @@
 ---
 - hosts: all
   vars:
-    kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory"
     nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo"
     nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple"
     nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}"
@@ -12,23 +11,6 @@
     nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/"
     nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona"
   tasks:
-    - name: generate kolla inventory file
-      copy:
-        dest: "{{ kolla_inventory_path }}"
-        content: |
-          {% for host in hostvars %}
-          {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }}
-          {% endfor %}
-
-    - name: setup logging
-      shell:
-        cmd: |
-          mkdir logs
-          ln -s $(pwd)/logs /tmp/logs
-          mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs}
-        executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
     - name: copy setup script
       copy:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh"
@@ -57,21 +39,12 @@
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
         dest: /etc/kolla/kolla-build.conf
 
-    - name: Template override
+    - name: Template template_overrides.j2
       template:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2"
         dest: /etc/kolla/template_overrides.j2
 
-    - shell:
-        cmd: |
-          set -e
-          set -x
-
-          export ACTION={{ action }}
-          export BASE_DISTRO={{ base_distro }}
-          export INSTALL_TYPE={{ install_type }}
-          export IN_PROJECT_JOBS=1
-          tools/gate_run.sh
-        executable: /bin/bash
+    - name: Run tox
+      command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }}
+      args:
         chdir: "{{ zuul.project.src_dir }}"
-      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh
deleted file mode 120000
index fbacb3f92..000000000
--- a/tests/setup_gate.sh
+++ /dev/null
@@ -1 +0,0 @@
-../tools/setup_gate.sh
\ No newline at end of file
diff --git a/tests/templates/kolla-build.conf.j2 b/tests/templates/kolla-build.conf.j2
index 3f9e5d04c..37b06aca6 100644
--- a/tests/templates/kolla-build.conf.j2
+++ b/tests/templates/kolla-build.conf.j2
@@ -5,5 +5,5 @@ namespace = kolla
 tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }}
 {% endif %}
 push = false
-logs_dir = /tmp/logs/build
+logs_dir = {{ kolla_build_logs_dir }}
 template_override = /etc/kolla/template_overrides.j2
diff --git a/tools/dump_info.sh b/tools/dump_info.sh
index b0b3ce3f1..26158653b 100755
--- a/tools/dump_info.sh
+++ b/tools/dump_info.sh
@@ -23,4 +23,4 @@ function dump_node_info {
     set -o errexit
 }
 
-(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s)
+(dump_node_info 2>&1) > node_info_$(date +%s)
diff --git a/tools/gate_run.sh b/tools/gate_run.sh
deleted file mode 100755
index 07f5e3574..000000000
--- a/tools/gate_run.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o xtrace
-
-# exist when the jobs is not project jobs.
-if [[ -z $IN_PROJECT_JOBS ]]; then
-    exit 0
-fi
-
-tools/setup_gate.sh
-tox -e $ACTION-$BASE_DISTRO-$INSTALL_TYPE
-
-if [[ -n $PACK_REGISTRY ]] && [[ $ACTION == "build" ]]; then
-    pack_registry
-fi
diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh
deleted file mode 100755
index e2c617dd0..000000000
--- a/tools/setup_gate.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -o xtrace
-set -o errexit
-
-# Enable unbuffered output for Ansible in Jenkins.
-export PYTHONUNBUFFERED=1
-
-function setup_registry {
-    sudo mkdir /opt/kolla_registry
-    sudo chmod -R 644 /opt/kolla_registry
-    docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2
-}
-
-setup_registry
diff --git a/tox.ini b/tox.ini
index 5d4e9902d..a448970ac 100644
--- a/tox.ini
+++ b/tox.ini
@@ -75,11 +75,6 @@ commands =
   doc8 doc
   sphinx-build -W -b html doc/source doc/build/html
 
-[testenv:setupenv]
-commands =
-  {toxinidir}/tools/setup_gate.sh
-  {toxinidir}/tools/dump_info.sh
-
 [testenv:build-centos-binary]
 whitelist_externals = find
                       bash
@@ -89,7 +84,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestCentosBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-centos-source]
 whitelist_externals = find
@@ -100,7 +94,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestCentosSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-ubuntu-binary]
 whitelist_externals = find
@@ -111,7 +104,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestUbuntuBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-ubuntu-source]
 whitelist_externals = find
@@ -122,7 +114,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestUbuntuSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-debian-binary]
 whitelist_externals = find
@@ -133,7 +124,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestDebianBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-debian-source]
 whitelist_externals = find
@@ -144,7 +134,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestDebianSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-oraclelinux-binary]
 whitelist_externals = find
@@ -155,7 +144,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestOracleLinuxBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-oraclelinux-source]
 whitelist_externals = find
@@ -166,7 +154,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestOracleLinuxSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:genconfig]
 whitelist_externals = which