Merge "Clean up test process"

5 files changed, 58 insertions, 85 deletions

diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml
index bb514821d..00f71f6b9 100644
--- a/tests/playbooks/post.yml
+++ b/tests/playbooks/post.yml
@@ -1,60 +1,66 @@
 ---
 - hosts: all
   tasks:
-    - shell:
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Collect various info to logs
+      shell:
         cmd: |
           set +o errexit
-          mkdir -p logs
-          # copy system log
+          set -o xtrace
 
-          sudo cp -r /var/log logs/system_log
+          # copy system log
+          sudo cp -r /var/log system_log
 
           if which journalctl ; then
               # the journal gives us syslog() and kernel output, so is like
               # a concatenation of the above.
-              sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
-              sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
+              sudo journalctl --no-pager > syslog.txt
+              sudo journalctl --no-pager -u docker.service > docker.log
           else
               # assume rsyslog
-              sudo cp /var/log/syslog logs/syslog.txt
-              sudo cp /var/log/kern.log logs/kern_log.txt
-              sudo cp /var/log/upstart/docker.log logs/docker.log
+              sudo cp /var/log/syslog syslog.txt
+              sudo cp /var/log/kern.log kern_log.txt
+              sudo cp /var/log/upstart/docker.log docker.log
           fi
 
           if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
-              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
+              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs
           fi
 
           # sudo config
-          sudo cp -r /etc/sudoers.d logs/
-          sudo cp /etc/sudoers logs/sudoers.txt
+          sudo cp -r /etc/sudoers.d ./
+          sudo cp /etc/sudoers sudoers.txt
 
-          df -h > logs/df.txt
-          free  > logs/free.txt
-          sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
-          mount > logs/mount.txt
-          env > logs/env.txt
+          df -h > df.txt
+          free  > free.txt
+          sudo parted -l > parted-l.txt
+          mount > mount.txt
+          env > env.txt
 
           if [ `command -v dpkg` ]; then
-              dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
+              dpkg -l > dpkg-l.txt
           fi
           if [ `command -v rpm` ]; then
-              rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
+              rpm -qa > rpm-qa.txt
           fi
 
           # final memory usage and process list
-          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
+          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt
 
           # docker related information
-          (docker info && docker system df && docker system df -v) > logs/docker-info.txt
+          (docker info && docker system df && docker system df -v) > docker-info.txt
 
-          sudo cp -r /etc/kolla logs/kolla_configs
+          sudo cp -r /etc/kolla kolla_configs
 
-          # fix the permissions for logs folder
-          sudo chmod -R 777 logs
+          # last sudo to own all files
+          sudo chown -R {{ ansible_user_id }}: .
 
           # Parse build logs and extract pkg install info
-          tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt
+          {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt
 
           # rename files to .txt; this is so that when displayed via
           # logs.openstack.org clicking results in the browser shows the
@@ -62,30 +68,21 @@
           # download it, etc.
 
           # firstly, rename all .log files to .txt files
-          for f in $(find logs -name "*.log"); do
-              sudo mv $f ${f/.log/.txt}
+          for f in $(find . -name "*.log"); do
+              mv $f ${f/.log/.txt}
           done
 
           # Update symlinks to new file names
-          for f in $(find logs -name "*FAILED*"); do
-              sudo mv ${f} ${f}.gz
-              sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
+          for f in $(find . -name "*FAILED*"); do
+              mv ${f} ${f}.gz
+              ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
           done
 
           # append .txt to all kolla config file
-          find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
+          find kolla_configs -type f -exec mv '{}' '{}'.txt \;
 
           # Compress all text logs
-          find logs -iname '*.txt' -execdir gzip -9 {} \+
-          find logs -iname '*.json' -execdir gzip -9 {} \+
+          find . -iname '*.txt' -execdir gzip -9 {} \+
+          find . -iname '*.json' -execdir gzip -9 {} \+
         executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
-    - synchronize:
-        src: '{{ zuul.project.src_dir }}/logs'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - "--quiet"
+        chdir: "{{ kolla_logs_dir }}"
diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml
index 00d471614..e721d8859 100644
--- a/tests/playbooks/pre.yml
+++ b/tests/playbooks/pre.yml
@@ -1,13 +1,17 @@
 ---
 - hosts: all
-  become: true
   tasks:
-    - name: Install dbus for debian system
-      apt: name=dbus
-      when: ansible_os_family == 'Debian'
+    - name: Create dir for kolla logs
+      file:
+        path: "{{ kolla_logs_dir }}"
+        state: directory
 
-    - name: Install ansible
-      pip:
-        name: ansible
-      when:
-        - inventory_hostname == "primary"
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Create dir for kolla build logs
+      file:
+        path: "{{ kolla_build_logs_dir }}"
+        state: directory
diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml
index b81c4f5e3..671b7282c 100644
--- a/tests/playbooks/run.yml
+++ b/tests/playbooks/run.yml
@@ -1,7 +1,6 @@
 ---
 - hosts: all
   vars:
-    kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory"
     nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo"
     nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple"
     nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}"
@@ -11,23 +10,6 @@
     nodepool_cbs_centos_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/cbs.centos"
     nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona"
   tasks:
-    - name: generate kolla inventory file
-      copy:
-        dest: "{{ kolla_inventory_path }}"
-        content: |
-          {% for host in hostvars %}
-          {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }}
-          {% endfor %}
-
-    - name: setup logging
-      shell:
-        cmd: |
-          mkdir logs
-          ln -s $(pwd)/logs /tmp/logs
-          mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs}
-        executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
     - name: copy setup script
       copy:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh"
@@ -56,21 +38,12 @@
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
         dest: /etc/kolla/kolla-build.conf
 
-    - name: Template override
+    - name: Template template_overrides.j2
       template:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2"
         dest: /etc/kolla/template_overrides.j2
 
-    - shell:
-        cmd: |
-          set -e
-          set -x
-
-          export ACTION={{ action }}
-          export BASE_DISTRO={{ base_distro }}
-          export INSTALL_TYPE={{ install_type }}
-          export IN_PROJECT_JOBS=1
-          tools/gate_run.sh
-        executable: /bin/bash
+    - name: Run tox
+      command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }}
+      args:
         chdir: "{{ zuul.project.src_dir }}"
-      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh
deleted file mode 120000
index fbacb3f92..000000000
--- a/tests/setup_gate.sh
+++ /dev/null
@@ -1 +0,0 @@
-../tools/setup_gate.sh
\ No newline at end of file
diff --git a/tests/templates/kolla-build.conf.j2 b/tests/templates/kolla-build.conf.j2
index 3f9e5d04c..37b06aca6 100644
--- a/tests/templates/kolla-build.conf.j2
+++ b/tests/templates/kolla-build.conf.j2
@@ -5,5 +5,5 @@ namespace = kolla
 tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }}
 {% endif %}
 push = false
-logs_dir = /tmp/logs/build
+logs_dir = {{ kolla_build_logs_dir }}
 template_override = /etc/kolla/template_overrides.j2