diff options
| -rw-r--r-- | .zuul.d/base.yaml | 6 | ||||
| -rw-r--r-- | tests/playbooks/post.yml | 83 | ||||
| -rw-r--r-- | tests/playbooks/pre.yml | 22 | ||||
| -rw-r--r-- | tests/playbooks/run.yml | 35 | ||||
| l--------- | tests/setup_gate.sh | 1 | ||||
| -rw-r--r-- | tests/templates/kolla-build.conf.j2 | 2 | ||||
| -rwxr-xr-x | tools/dump_info.sh | 2 | ||||
| -rwxr-xr-x | tools/gate_run.sh | 16 | ||||
| -rwxr-xr-x | tools/setup_gate.sh | 15 | ||||
| -rw-r--r-- | tox.ini | 13 |
10 files changed, 62 insertions, 133 deletions
diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 0e2fbbec5..2fc3dcf08 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -46,9 +46,6 @@ run: tests/playbooks/run.yml post-run: tests/playbooks/post.yml attempts: 1 - required-projects: - - openstack/kolla-ansible - - openstack/requirements irrelevant-files: - ^.*\.rst$ - ^doc/.* @@ -59,3 +56,6 @@ - ^test-requirements.txt$ vars: publisher: false + extra-vars: + kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla" + kolla_build_logs_dir: "{{ kolla_logs_dir }}/build" diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml index bb514821d..00f71f6b9 100644 --- a/tests/playbooks/post.yml +++ b/tests/playbooks/post.yml @@ -1,60 +1,66 @@ --- - hosts: all tasks: - - shell: + - name: Dump host info to logs + command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + args: + chdir: "{{ kolla_logs_dir }}" + + - name: Collect various info to logs + shell: cmd: | set +o errexit - mkdir -p logs - # copy system log + set -o xtrace - sudo cp -r /var/log logs/system_log + # copy system log + sudo cp -r /var/log system_log if which journalctl ; then # the journal gives us syslog() and kernel output, so is like # a concatenation of the above. - sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null - sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null + sudo journalctl --no-pager > syslog.txt + sudo journalctl --no-pager -u docker.service > docker.log else # assume rsyslog - sudo cp /var/log/syslog logs/syslog.txt - sudo cp /var/log/kern.log logs/kern_log.txt - sudo cp /var/log/upstart/docker.log logs/docker.log + sudo cp /var/log/syslog syslog.txt + sudo cp /var/log/kern.log kern_log.txt + sudo cp /var/log/upstart/docker.log docker.log fi if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then - sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs + sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs fi # sudo config - sudo cp -r /etc/sudoers.d logs/ - sudo cp /etc/sudoers logs/sudoers.txt + sudo cp -r /etc/sudoers.d ./ + sudo cp /etc/sudoers sudoers.txt - df -h > logs/df.txt - free > logs/free.txt - sudo parted -l | sudo tee logs/parted-l.txt > /dev/null - mount > logs/mount.txt - env > logs/env.txt + df -h > df.txt + free > free.txt + sudo parted -l > parted-l.txt + mount > mount.txt + env > env.txt if [ `command -v dpkg` ]; then - dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null + dpkg -l > dpkg-l.txt fi if [ `command -v rpm` ]; then - rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null + rpm -qa > rpm-qa.txt fi # final memory usage and process list - ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt + ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt # docker related information - (docker info && docker system df && docker system df -v) > logs/docker-info.txt + (docker info && docker system df && docker system df -v) > docker-info.txt - sudo cp -r /etc/kolla logs/kolla_configs + sudo cp -r /etc/kolla kolla_configs - # fix the permissions for logs folder - sudo chmod -R 777 logs + # last sudo to own all files + sudo chown -R {{ ansible_user_id }}: . # Parse build logs and extract pkg install info - tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt + {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt # rename files to .txt; this is so that when displayed via # logs.openstack.org clicking results in the browser shows the @@ -62,30 +68,21 @@ # download it, etc. # firstly, rename all .log files to .txt files - for f in $(find logs -name "*.log"); do - sudo mv $f ${f/.log/.txt} + for f in $(find . -name "*.log"); do + mv $f ${f/.log/.txt} done # Update symlinks to new file names - for f in $(find logs -name "*FAILED*"); do - sudo mv ${f} ${f}.gz - sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz + for f in $(find . -name "*FAILED*"); do + mv ${f} ${f}.gz + ln -sf ${f#*/000_FAILED_}.gz ${f}.gz done # append .txt to all kolla config file - find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \; + find kolla_configs -type f -exec mv '{}' '{}'.txt \; # Compress all text logs - find logs -iname '*.txt' -execdir gzip -9 {} \+ - find logs -iname '*.json' -execdir gzip -9 {} \+ + find . -iname '*.txt' -execdir gzip -9 {} \+ + find . -iname '*.json' -execdir gzip -9 {} \+ executable: /bin/bash - chdir: "{{ zuul.project.src_dir }}" - - - synchronize: - src: '{{ zuul.project.src_dir }}/logs' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - "--quiet" + chdir: "{{ kolla_logs_dir }}" diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 00d471614..e721d8859 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -1,13 +1,17 @@ --- - hosts: all - become: true tasks: - - name: Install dbus for debian system - apt: name=dbus - when: ansible_os_family == 'Debian' + - name: Create dir for kolla logs + file: + path: "{{ kolla_logs_dir }}" + state: directory - - name: Install ansible - pip: - name: ansible - when: - - inventory_hostname == "primary" + - name: Dump host info to logs + command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + args: + chdir: "{{ kolla_logs_dir }}" + + - name: Create dir for kolla build logs + file: + path: "{{ kolla_build_logs_dir }}" + state: directory diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index b15ec45b9..0578dea32 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -1,7 +1,6 @@ --- - hosts: all vars: - kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory" nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo" nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple" nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}" @@ -12,23 +11,6 @@ nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/" nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona" tasks: - - name: generate kolla inventory file - copy: - dest: "{{ kolla_inventory_path }}" - content: | - {% for host in hostvars %} - {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }} - {% endfor %} - - - name: setup logging - shell: - cmd: | - mkdir logs - ln -s $(pwd)/logs /tmp/logs - mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs} - executable: /bin/bash - chdir: "{{ zuul.project.src_dir }}" - - name: copy setup script copy: src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh" @@ -57,21 +39,12 @@ src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2" dest: /etc/kolla/kolla-build.conf - - name: Template override + - name: Template template_overrides.j2 template: src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2" dest: /etc/kolla/template_overrides.j2 - - shell: - cmd: | - set -e - set -x - - export ACTION={{ action }} - export BASE_DISTRO={{ base_distro }} - export INSTALL_TYPE={{ install_type }} - export IN_PROJECT_JOBS=1 - tools/gate_run.sh - executable: /bin/bash + - name: Run tox + command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }} + args: chdir: "{{ zuul.project.src_dir }}" - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh deleted file mode 120000 index fbacb3f92..000000000 --- a/tests/setup_gate.sh +++ /dev/null @@ -1 +0,0 @@ -../tools/setup_gate.sh
\ No newline at end of file diff --git a/tests/templates/kolla-build.conf.j2 b/tests/templates/kolla-build.conf.j2 index 3f9e5d04c..37b06aca6 100644 --- a/tests/templates/kolla-build.conf.j2 +++ b/tests/templates/kolla-build.conf.j2 @@ -5,5 +5,5 @@ namespace = kolla tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }} {% endif %} push = false -logs_dir = /tmp/logs/build +logs_dir = {{ kolla_build_logs_dir }} template_override = /etc/kolla/template_overrides.j2 diff --git a/tools/dump_info.sh b/tools/dump_info.sh index b0b3ce3f1..26158653b 100755 --- a/tools/dump_info.sh +++ b/tools/dump_info.sh @@ -23,4 +23,4 @@ function dump_node_info { set -o errexit } -(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s) +(dump_node_info 2>&1) > node_info_$(date +%s) diff --git a/tools/gate_run.sh b/tools/gate_run.sh deleted file mode 100755 index 07f5e3574..000000000 --- a/tools/gate_run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o xtrace - -# exist when the jobs is not project jobs. -if [[ -z $IN_PROJECT_JOBS ]]; then - exit 0 -fi - -tools/setup_gate.sh -tox -e $ACTION-$BASE_DISTRO-$INSTALL_TYPE - -if [[ -n $PACK_REGISTRY ]] && [[ $ACTION == "build" ]]; then - pack_registry -fi diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh deleted file mode 100755 index e2c617dd0..000000000 --- a/tools/setup_gate.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -# Enable unbuffered output for Ansible in Jenkins. -export PYTHONUNBUFFERED=1 - -function setup_registry { - sudo mkdir /opt/kolla_registry - sudo chmod -R 644 /opt/kolla_registry - docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2 -} - -setup_registry @@ -75,11 +75,6 @@ commands = doc8 doc sphinx-build -W -b html doc/source doc/build/html -[testenv:setupenv] -commands = - {toxinidir}/tools/setup_gate.sh - {toxinidir}/tools/dump_info.sh - [testenv:build-centos-binary] whitelist_externals = find bash @@ -89,7 +84,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestCentosBinary - {toxinidir}/tools/dump_info.sh [testenv:build-centos-source] whitelist_externals = find @@ -100,7 +94,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestCentosSource - {toxinidir}/tools/dump_info.sh [testenv:build-ubuntu-binary] whitelist_externals = find @@ -111,7 +104,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestUbuntuBinary - {toxinidir}/tools/dump_info.sh [testenv:build-ubuntu-source] whitelist_externals = find @@ -122,7 +114,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestUbuntuSource - {toxinidir}/tools/dump_info.sh [testenv:build-debian-binary] whitelist_externals = find @@ -133,7 +124,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestDebianBinary - {toxinidir}/tools/dump_info.sh [testenv:build-debian-source] whitelist_externals = find @@ -144,7 +134,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestDebianSource - {toxinidir}/tools/dump_info.sh [testenv:build-oraclelinux-binary] whitelist_externals = find @@ -155,7 +144,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestOracleLinuxBinary - {toxinidir}/tools/dump_info.sh [testenv:build-oraclelinux-source] whitelist_externals = find @@ -166,7 +154,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestOracleLinuxSource - {toxinidir}/tools/dump_info.sh [testenv:genconfig] whitelist_externals = which |