Merge "Moving jobs to kolla repo"

10 files changed, 254 insertions, 241 deletions

diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 000000000..822a46d8e
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,96 @@
+- project:
+    name: openstack/kolla
+    check:
+      jobs:
+        - kolla-build-centos-binary
+        - kolla-build-centos-source
+        - kolla-build-ubuntu-binary
+        - kolla-build-ubuntu-source
+        - kolla-build-oraclelinux-binary
+        - kolla-build-oraclelinux-source
+    gate:
+      jobs:
+        - kolla-build-centos-source
+        - kolla-build-ubuntu-source
+        - kolla-build-oraclelinux-source
+
+- nodeset:
+    name: kolla-centos
+    nodes:
+      - name: primary
+        label: centos-7
+
+- nodeset:
+    name: kolla-ubuntu
+    nodes:
+      - name: primary
+        label: ubuntu-xenial
+
+- job:
+    name: kolla-base
+    parent: base
+    timeout: 5400
+    pre-run: tests/playbooks/pre
+    run: tests/playbooks/run
+    post-run: tests/playbooks/post
+    attempts: 1
+    required-projects:
+      - openstack/kolla-ansible
+      - openstack/requirements
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+
+- job:
+    name: kolla-build-centos-binary
+    parent: kolla-base
+    nodeset: kolla-centos
+    vars:
+      action: build
+      base_distro: centos
+      install_type: binary
+
+- job:
+    name: kolla-build-centos-source
+    parent: kolla-base
+    nodeset: kolla-centos
+    vars:
+      action: build
+      base_distro: centos
+      install_type: source
+
+- job:
+    name: kolla-build-ubuntu-binary
+    parent: kolla-base
+    nodeset: kolla-ubuntu
+    vars:
+      action: build
+      base_distro: ubuntu
+      install_type: binary
+
+- job:
+    name: kolla-build-ubuntu-source
+    parent: kolla-base
+    nodeset: kolla-ubuntu
+    vars:
+      action: build
+      base_distro: ubuntu
+      install_type: source
+
+- job:
+    name: kolla-build-oraclelinux-binary
+    parent: kolla-base
+    nodeset: kolla-centos
+    vars:
+      action: build
+      base_distro: oraclelinux
+      install_type: binary
+
+- job:
+    name: kolla-build-oraclelinux-source
+    parent: kolla-base
+    nodeset: kolla-centos
+    vars:
+      action: build
+      base_distro: oraclelinux
+      install_type: source
diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml
new file mode 100644
index 000000000..345ccab04
--- /dev/null
+++ b/tests/playbooks/post.yml
@@ -0,0 +1,79 @@
+- hosts: all
+  tasks:
+    - shell:
+        cmd: |
+          set +o errexit
+          mkdir -p logs
+          # copy system log
+
+          sudo cp -r /var/log logs/system_log
+
+          if which journalctl ; then
+              # the journal gives us syslog() and kernel output, so is like
+              # a concatenation of the above.
+              sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
+              sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
+          else
+              # assume rsyslog
+              sudo cp /var/log/syslog logs/syslog.txt
+              sudo cp /var/log/kern.log logs/kern_log.txt
+              sudo cp /var/log/upstart/docker.log logs/docker.log
+          fi
+
+          if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
+              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
+          fi
+
+          # sudo config
+          sudo cp -r /etc/sudoers.d logs/
+          sudo cp /etc/sudoers logs/sudoers.txt
+
+          df -h > logs/df.txt
+          free  > logs/free.txt
+          sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
+          mount > logs/mount.txt
+          env > logs/env.txt
+
+          if [ `command -v dpkg` ]; then
+              dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
+          fi
+          if [ `command -v rpm` ]; then
+              rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
+          fi
+
+          # final memory usage and process list
+          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
+
+          # docker related information
+          (docker info && docker images && docker ps -a) > logs/docker-info.txt
+
+          sudo cp -r /etc/kolla logs/kolla_configs
+
+          # fix the permissions for logs folder
+          sudo chmod -R 777 logs
+
+          # rename files to .txt; this is so that when displayed via
+          # logs.openstack.org clicking results in the browser shows the
+          # files, rather than trying to send it to another app or make you
+          # download it, etc.
+
+          # firstly, rename all .log files to .txt files
+          for f in $(find logs -name "*.log"); do
+              sudo mv $f ${f/.log/.txt}
+          done
+
+          # append .txt to all kolla config file
+          find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
+
+          # Compress all text logs
+          find logs -iname '*.txt' -execdir gzip -9 {} \+
+          find logs -iname '*.json' -execdir gzip -9 {} \+
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - synchronize:
+        src: '{{ zuul.project.src_dir }}/logs'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml
new file mode 100644
index 000000000..4be5f1375
--- /dev/null
+++ b/tests/playbooks/pre.yml
@@ -0,0 +1,13 @@
+- hosts: all
+  become: true
+
+  tasks:
+    - name: Install dbus for debian system
+      apt: name=dbus
+      when: ansible_os_family == 'Debian'
+
+    - name: Install ansible
+      pip:
+        name: ansible
+      when:
+        - inventory_hostname == "primary"
diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml
new file mode 100644
index 000000000..5d08a21a5
--- /dev/null
+++ b/tests/playbooks/run.yml
@@ -0,0 +1,51 @@
+- hosts: all
+  vars:
+    kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory"
+  tasks:
+    - name: generate kolla inventory file
+      copy:
+        dest: "{{ kolla_inventory_path }}"
+        content: |
+          {% for host in hostvars %}
+          {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }}
+          {% endfor %}
+
+    - name: setup logging
+      shell:
+        cmd: |
+          mkdir logs
+          ln -s $(pwd)/logs /tmp/logs
+          mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs,system_logs}
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: copy setup script
+      copy:
+        src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh"
+        dest: /tmp/setup.sh
+        mode: 0755
+
+    - name: run node setup script
+      shell: /tmp/setup.sh
+      become: true
+
+    - name: changing permission of Docker socket to 666
+      file:
+        path: /run/docker.sock
+        mode: 666
+      become: true
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+
+          export ACTION={{ action }}
+          export BASE_DISTRO={{ base_distro }}
+          export INSTALL_TYPE={{ install_type }}
+          export IN_PROJECT_JOBS=1
+
+          tools/gate_run.sh
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/tools/deploy_aio.sh b/tools/deploy_aio.sh
index 56134eb8c..aee1118a2 100755
--- a/tools/deploy_aio.sh
+++ b/tools/deploy_aio.sh
@@ -22,16 +22,16 @@ EOF
 pushd "${KOLLA_ANSIBLE_DIR}"
 
 function get_logs {
-    sudo ansible-playbook -i ${RAW_INVENTORY} --become ${KOLLA_ANSIBLE_DIR}/tests/ansible_get_logs.yml > /tmp/logs/ansible/get-logs
+    sudo ansible-playbook -i ${INVENTORY} --become ${KOLLA_ANSIBLE_DIR}/tests/ansible_get_logs.yml > /tmp/logs/ansible/get-logs
 }
 
 # Copy configs
 sudo cp -a etc/kolla /etc/
 # Generate passwords
-export RAW_INVENTORY=/tmp/kolla/raw_inventory
+export INVENTORY=/tmp/kolla/inventory
 
-sudo ansible-playbook -i ${RAW_INVENTORY} --become tests/ansible_generate_inventory.yml
-sudo ansible-playbook -i ${RAW_INVENTORY} --become -e action="deploy" -e type=$KOLLA_TYPE -e base=$KOLLA_BASE tests/ansible_generate_config.yml > /tmp/logs/ansible/generate_config
+sudo ansible-playbook -i ${INVENTORY} --become tests/ansible_generate_inventory.yml
+sudo ansible-playbook -i ${INVENTORY} --become -e action="deploy" -e type=$KOLLA_TYPE -e base=$KOLLA_BASE tests/ansible_generate_config.yml > /tmp/logs/ansible/generate_config
 
 trap get_logs EXIT
 
@@ -39,9 +39,9 @@ sudo ip l a fake_interface type dummy
 
 sudo tools/generate_passwords.py
 sudo chmod -R 777 /etc/kolla
-sudo tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks
-sudo tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
-sudo tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
+sudo tools/kolla-ansible -i ${INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks
+sudo tools/kolla-ansible -i ${INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
+sudo tools/kolla-ansible -i ${INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
 
 get_logs
 
diff --git a/tools/gate_run.sh b/tools/gate_run.sh
index d863afa9a..07f5e3574 100755
--- a/tools/gate_run.sh
+++ b/tools/gate_run.sh
@@ -3,95 +3,10 @@
 set -o errexit
 set -o xtrace
 
-function collect_logs {
-    set +o errexit
-    mkdir -p logs
-
-    # copy system log
-
-    sudo cp -r /var/log logs/system_log
-
-    if which journalctl ; then
-        # the journal gives us syslog() and kernel output, so is like
-        # a concatenation of the above.
-        sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
-        sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
-    else
-        # assume rsyslog
-        sudo cp /var/log/syslog logs/syslog.txt
-        sudo cp /var/log/kern.log logs/kern_log.txt
-        sudo cp /var/log/upstart/docker.log logs/docker.log
-    fi
-
-    if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
-        sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
-    fi
-
-    # sudo config
-    sudo cp -r /etc/sudoers.d logs/
-    sudo cp /etc/sudoers logs/sudoers.txt
-
-    df -h > logs/df.txt
-    free  > logs/free.txt
-    sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
-    mount > logs/mount.txt
-    env > logs/env.txt
-
-    if [ `command -v dpkg` ]; then
-        dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
-    fi
-    if [ `command -v rpm` ]; then
-        rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
-    fi
-
-    # final memory usage and process list
-    ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
-
-    # docker related information
-    (docker info && docker images && docker ps -a) > logs/docker-info.txt
-
-    sudo cp -r /etc/kolla logs/kolla_configs
-
-    # fix the permissions for logs folder
-    sudo chmod -R 777 logs
-
-    # rename files to .txt; this is so that when displayed via
-    # logs.openstack.org clicking results in the browser shows the
-    # files, rather than trying to send it to another app or make you
-    # download it, etc.
-
-    # firstly, rename all .log files to .txt files
-    for f in $(find logs -name "*.log"); do
-        sudo mv $f ${f/.log/.txt}
-    done
-
-    # append .txt to all kolla config file
-    find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
-
-    # Compress all text logs
-    find logs -iname '*.txt' -execdir gzip -9 {} \+
-    find logs -iname '*.json' -execdir gzip -9 {} \+
-
-    set -o errexit
-}
-
-function pack_registry {
-    sudo mkdir "images"
-    if [ -n "$ZUUL_BRANCH" ]; then
-        BRANCH=$(echo $ZUUL_BRANCH | cut -d/ -f2)
-    else
-        BRANCH=$(echo $ZUUL_REFNAME | cut -d/ -f2)
-    fi
-
-    FILENAME=${BASE_DISTRO}-${INSTALL_TYPE}-registry-${BRANCH}.tar.gz
-    sudo docker stop registry
-    sudo tar -zcf "images/$FILENAME" -C /opt/kolla_registry .
-    sudo docker start registry
-    sudo chmod 755 -R images
-}
-
-
-trap collect_logs EXIT
+# exist when the jobs is not project jobs.
+if [[ -z $IN_PROJECT_JOBS ]]; then
+    exit 0
+fi
 
 tools/setup_gate.sh
 tox -e $ACTION-$BASE_DISTRO-$INSTALL_TYPE
diff --git a/tools/playbook-setup-nodes.yml b/tools/playbook-setup-nodes.yml
deleted file mode 100644
index 19bdf56b4..000000000
--- a/tools/playbook-setup-nodes.yml
+++ /dev/null
@@ -1,41 +0,0 @@
----
-- hosts: all
-  become: true
-
-  tasks:
-  - name: Setup /etc/hosts
-    copy:
-      src: /etc/hosts
-      dest: /etc/hosts
-
-  - name: Ensure /etc/hostname is valid for SELinux
-    command: restorecon -v /etc/hostname
-    when: ansible_os_family == 'RedHat'
-
-  - name: Assign hostname
-    hostname:
-      name: "{{ inventory_hostname }}"
-
-  - name: Copy setup script
-    copy:
-      src: setup_{{ ansible_os_family }}.sh
-      dest: /tmp/setup.sh
-      mode: 0755
-
-- hosts: all
-  become: true
-
-  tasks:
-  - name: Create log directory for node
-    file:
-      state: directory
-      path: /tmp/{{ inventory_hostname }}
-    become: false
-
-  - name: Run node setup
-    shell: /tmp/setup.sh
-
-  - name: Changing permissions of Docker socket to 666
-    file:
-      path: /run/docker.sock
-      mode: 0666
diff --git a/tools/setup_Debian.sh b/tools/setup_Debian.sh
index f5fd7ed43..1ba873185 100755
--- a/tools/setup_Debian.sh
+++ b/tools/setup_Debian.sh
@@ -68,7 +68,7 @@ sudo service docker stop
 if [[ ${DISTRIB_CODENAME} == "trusty" ]]; then
     sudo apt-get -y install --no-install-recommends btrfs-tools
     setup_disk
-    echo "DOCKER_OPTS=\"-s btrfs --insecure-registry $(cat /etc/nodepool/primary_node_private):4000\"" | sudo tee /etc/default/docker
+    echo "DOCKER_OPTS=\"-s btrfs --insecure-registry 0.0.0.0/0\"" | sudo tee /etc/default/docker
     sudo mount --make-shared /run
     sudo service docker start
 else
@@ -76,7 +76,7 @@ else
     sudo tee /etc/systemd/system/docker.service.d/kolla.conf << EOF
 [Service]
 ExecStart=
-ExecStart=/usr/bin/dockerd --storage-driver overlay2 --insecure-registry $(cat /etc/nodepool/primary_node_private):4000
+ExecStart=/usr/bin/dockerd --storage-driver overlay2 --insecure-registry 0.0.0.0/0
 MountFlags=shared
 EOF
     sudo systemctl daemon-reload
diff --git a/tools/setup_RedHat.sh b/tools/setup_RedHat.sh
index 2bd3bb09f..acb6c3a23 100755
--- a/tools/setup_RedHat.sh
+++ b/tools/setup_RedHat.sh
@@ -70,7 +70,7 @@ sudo mkdir /etc/systemd/system/docker.service.d
 sudo tee /etc/systemd/system/docker.service.d/kolla.conf << EOF
 [Service]
 ExecStart=
-ExecStart=/usr/bin/dockerd --storage-driver btrfs --insecure-registry=$(cat /etc/nodepool/primary_node_private):4000
+ExecStart=/usr/bin/dockerd --storage-driver btrfs --insecure-registry=0.0.0.0/0
 MountFlags=shared
 EOF
 
diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh
index 4241012ca..a0951a8d7 100755
--- a/tools/setup_gate.sh
+++ b/tools/setup_gate.sh
@@ -6,14 +6,8 @@ set -o errexit
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
-. /etc/nodepool/provider
 . /etc/ci/mirror_info.sh
 
-# Just for mandre :)
-if [[ ! -f /etc/sudoers.d/jenkins ]]; then
-    echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins
-fi
-
 function setup_config {
 
     sudo mkdir -p /etc/kolla
@@ -87,108 +81,14 @@ registry = 127.0.0.1:4000
 push = true
 logs_dir = /tmp/logs/build
 EOF
-
-}
-
-function detect_distro {
-    DISTRO=$(ansible all -i "localhost," -msetup -clocal | awk -F\" '/ansible_os_family/ {print $4}')
-}
-
-# NOTE(sdake): This works around broken nodepool on systems with only one
-#              private interface
-#              The big regex checks for IP addresses in the file
-function setup_workaround_broken_nodepool {
-    if [[ `grep -E -o "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" /etc/nodepool/node_private | wc -l` -eq 0 ]]; then
-        cp /etc/nodepool/node /etc/nodepool/node_private
-        cp /etc/nodepool/sub_nodes /etc/nodepool/sub_nodes_private
-    fi
-}
-
-function setup_ssh {
-    # Generate a new keypair that Ansible will use
-    ssh-keygen -f /home/jenkins/.ssh/kolla -N ''
-    cat /home/jenkins/.ssh/kolla.pub >> /home/jenkins/.ssh/authorized_keys
-
-    # Push the public key around to all of the nodes
-    for ip in $(cat /etc/nodepool/sub_nodes_private); do
-        scp /home/jenkins/.ssh/kolla.pub ${ip}:/home/jenkins/.ssh/authorized_keys
-        # TODO(SamYaple): Remove this root key pushing once Kolla doesn't
-        # require root anymore.
-        ssh ${ip} -i /home/jenkins/.ssh/kolla 'sudo mkdir -p /root/.ssh; sudo cp /home/jenkins/.ssh/* /root/.ssh/'
-    done
-
-    # From now on use the new IdentityFile for connecting to other hosts
-    echo "IdentityFile /home/jenkins/.ssh/kolla" >> /home/jenkins/.ssh/config
-    chmod 600 /home/jenkins/.ssh/config
-}
-
-function setup_inventory {
-    local counter=0
-
-    echo -e "127.0.0.1\tlocalhost" > /tmp/hosts
-    for ip in $(cat /etc/nodepool/{node_private,sub_nodes_private}); do
-        : $((counter++))
-        # FIXME(jeffrey4l): do not set two hostnames in oneline. this is a
-        # wordround fix for the rabbitmq failed when deploy on CentOS in the CI
-        # gate. the ideal fix should set the hostname in setup_gate.sh script.
-        # But it do not work as expect with unknown reason
-        ssh-keyscan "${ip}" >> ~/.ssh/known_hosts
-        echo -e "${ip}\tnode${counter}" >> /tmp/hosts
-        echo -e "${ip}\t$(ssh ${ip} hostname)" >> /tmp/hosts
-        echo "node${counter}" >> ${RAW_INVENTORY}
-    done
-
-    sudo chown root: /tmp/hosts
-    sudo chmod 644 /tmp/hosts
-    sudo mv /tmp/hosts /etc/hosts
-}
-
-function setup_ansible {
-    RAW_INVENTORY=/tmp/kolla/raw_inventory
-    mkdir /tmp/kolla
-
-    # TODO(SamYaple): Move to virtualenv
-    sudo -H pip install -U "ansible>=2,<2.4" "docker>=2.0.0" "python-openstackclient" "python-neutronclient" "ara"
-    detect_distro
-
-    setup_inventory
-
-    sudo mkdir /etc/ansible
-    sudo tee /etc/ansible/ansible.cfg<<EOF
-[defaults]
-callback_plugins = /usr/lib/python2.7/site-packages/ara/plugins/callbacks:\$VIRTUAL_ENV/lib/python2.7/site-packages/ara/plugins/callbacks
-host_key_checking = False
-EOF
-
-    # Record the running state of the environment as seen by the setup module
-    ansible all -i ${RAW_INVENTORY} -m setup > /tmp/logs/ansible/initial-setup
-}
-
-function setup_node {
-    ansible-playbook -i ${RAW_INVENTORY} tools/playbook-setup-nodes.yml
-}
-
-function setup_logging {
-    # This directory is the directory that is copied with the devstack-logs
-    # publisher. It must exist at /home/jenkins/workspace/<job-name>/logs
-    mkdir logs
-
-    # For ease of access we symlink that logs directory to a known path
-    ln -s $(pwd)/logs /tmp/logs
-    mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs,system_logs}
 }
 
 function setup_registry {
+
     sudo mkdir /opt/kolla_registry
     sudo chmod -R 644 /opt/kolla_registry
     docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2
 }
 
-setup_logging
-tools/dump_info.sh
-setup_workaround_broken_nodepool
-setup_ssh
-setup_ansible
-setup_node
 setup_registry
 setup_config