Merge "Convert gate to file backend rather than partition"

6 files changed, 288 insertions, 0 deletions

diff --git a/tools/deploy_aio.sh b/tools/deploy_aio.sh
new file mode 100755
index 000000000..9ca10164b
--- /dev/null
+++ b/tools/deploy_aio.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+function print_failure {
+    docker ps -a
+    for failed in $(docker ps -a --format "{{.Names}}" --filter status=exited); do
+        docker logs --tail=all $failed
+    done
+    echo "FAILED"
+    exit 1
+}
+
+# Populate globals.yml
+cat << EOF > /etc/kolla/globals.yml
+---
+kolla_base_distro: "$1"
+kolla_install_type: "$2"
+kolla_internal_address: "169.254.169.10"
+docker_restart_policy: "never"
+network_interface: "eth0"
+neutron_external_interface: "fake_interface"
+EOF
+
+# Create dummy interface for neutron
+ip l a fake_interface type dummy
+
+# Actually do the deployment
+tools/kolla-ansible deploy || print_failure
+
+# TODO(SamYaple): Actually validate that all containers are started
+docker ps -a
+
+# TODO(SamYaple): Actually do functional testing of OpenStack
diff --git a/tools/dump_info.sh b/tools/dump_info.sh
new file mode 100755
index 000000000..b0b3ce3f1
--- /dev/null
+++ b/tools/dump_info.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -o xtrace
+
+function dump_node_info {
+    # NOTE(SamYaple): function for debugging gate
+    set +o errexit
+    local OLD_PATH="${PATH}"
+    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    sudo parted -l
+
+    sudo mount
+
+    df -h
+
+    uname -a
+
+    cat /etc/*release*
+
+    cat /proc/meminfo
+
+    PATH="${OLD_PATH}"
+    set -o errexit
+}
+
+(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s)
diff --git a/tools/setup_Debian.sh b/tools/setup_Debian.sh
new file mode 100755
index 000000000..f45b34ddc
--- /dev/null
+++ b/tools/setup_Debian.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+function setup_disk {
+    sudo swapoff -a
+    sudo dd if=/dev/zero of=/swapfile bs=1M count=4096
+    sudo chmod 0600 /swapfile
+    sudo mkswap /swapfile
+    sudo /sbin/swapon /swapfile
+
+    sudo dd if=/dev/zero of=/docker bs=1M count=10240
+    losetup -f /docker
+    DEV=$(losetup -a | awk -F: '/\/docker/ {print $1}')
+
+    # Format Disks and setup Docker to use BTRFS
+    sudo parted ${DEV} -s -- mklabel msdos
+    sudo rm -rf /var/lib/docker
+    sudo mkdir /var/lib/docker
+
+    # We want to snapshot the entire docker directory so we have to first create a
+    # subvolume and use that as the root for the docker directory.
+    sudo mkfs.btrfs -f ${DEV}
+    sudo mount ${DEV} /var/lib/docker
+    sudo btrfs subvolume create /var/lib/docker/docker
+    sudo umount /var/lib/docker
+    sudo mount -o noatime,subvol=docker ${DEV} /var/lib/docker
+}
+
+# (SamYaple)TODO: Remove the path overriding
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+# Setup Docker repo and add signing key
+echo 'deb http://apt.dockerproject.org/repo ubuntu-trusty main' | sudo tee /etc/apt/sources.list.d/docker.list
+sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends docker-engine btrfs-tools
+
+sudo service docker stop
+setup_disk
+echo 'DOCKER_OPTS="-s btrfs"' | sudo tee /etc/default/docker
+sudo service docker start
+
+sudo docker info
+
+echo "Completed $0."
diff --git a/tools/setup_RedHat.sh b/tools/setup_RedHat.sh
new file mode 100755
index 000000000..a04638f58
--- /dev/null
+++ b/tools/setup_RedHat.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+function setup_disk {
+    sudo swapoff -a
+    sudo dd if=/dev/zero of=/swapfile bs=1M count=4096
+    sudo chmod 0600 /swapfile
+    sudo mkswap /swapfile
+    sudo /sbin/swapon /swapfile
+
+    sudo dd if=/dev/zero of=/docker bs=1M count=10240
+    losetup -f /docker
+    DEV=$(losetup -a | awk -F: '/\/docker/ {print $1}')
+
+    # Format Disks and setup Docker to use BTRFS
+    sudo parted ${DEV} -s -- mklabel msdos
+    sudo rm -rf /var/lib/docker
+    sudo mkdir /var/lib/docker
+
+    # We want to snapshot the entire docker directory so we have to first create a
+    # subvolume and use that as the root for the docker directory.
+    sudo mkfs.btrfs -f ${DEV}
+    sudo mount ${DEV} /var/lib/docker
+    sudo btrfs subvolume create /var/lib/docker/docker
+    sudo umount /var/lib/docker
+    sudo mount -o noatime,subvol=docker ${DEV} /var/lib/docker
+}
+
+# (SamYaple)TODO: Remove the path overriding
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+cat | sudo tee /etc/yum.repos.d/docker.repo << EOF
+[docker]
+name=Docker Main Repository
+baseurl=https://yum.dockerproject.org/repo/main/centos/7
+enabled=1
+gpgcheck=1
+gpgkey=https://yum.dockerproject.org/gpg
+EOF
+
+sudo yum install -y libffi-devel openssl-devel docker-engine btrfs-progs
+
+setup_disk
+
+# Setup Docker
+sudo sed -i -r 's,(ExecStart)=(.+),\1=/usr/bin/docker daemon --storage-driver btrfs,' /usr/lib/systemd/system/docker.service
+sudo systemctl daemon-reload
+
+sudo systemctl start docker
+sudo docker info
+
+# disable ipv6 until we're sure routes to fedora mirrors work properly
+sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf'
+sudo /usr/sbin/sysctl -p
+
+echo "Completed $0."
diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh
new file mode 100755
index 000000000..79ea00a65
--- /dev/null
+++ b/tools/setup_gate.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Just for mandre :)
+if [[ ! -f /etc/sudoers.d/jenkins ]]; then
+    echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins
+fi
+
+function setup_config {
+    # generate the config
+    tox -e genconfig
+    # Copy configs
+    sudo cp -a etc/kolla /etc/
+}
+
+function detect_distro {
+    DISTRO=$(ansible all -i "localhost," -msetup -clocal | awk -F\" '/ansible_os_family/ {print $4}')
+}
+
+function setup_ssh {
+    # Generate a new keypair that Ansible will use
+    ssh-keygen -f /home/jenkins/.ssh/kolla -N ''
+    cat /home/jenkins/.ssh/kolla.pub >> /home/jenkins/.ssh/authorized_keys
+
+    # Push the the public key around to all of the nodes
+    for ip in $(cat /etc/nodepool/sub_nodes_private); do
+        scp /home/jenkins/.ssh/kolla.pub ${ip}:/home/jenkins/.ssh/authorized_keys
+        # TODO(SamYaple): Remove this root key pushing once Kolla doesn't
+        # require root anymore.
+        ssh ${ip} -i /home/jenkins/.ssh/kolla 'sudo mkdir -p /root/.ssh; sudo cp /home/jenkins/.ssh/* /root/.ssh/'
+    done
+
+    # From now on use the new IdentityFile for connecting to other hosts
+    echo "IdentityFile /home/jenkins/.ssh/kolla" >> /home/jenkins/.ssh/config
+}
+
+function setup_inventory {
+    local counter=0
+
+    detect_distro
+    if [[ "${DISTRO}" == "Debian" ]]; then
+        ANSIBLE_CONNECTION_TYPE=ssh
+    else
+        ANSIBLE_CONNECTION_TYPE=local
+    fi
+
+    echo -e "127.0.0.1\tlocalhost" > /tmp/hosts
+    for ip in $(cat /etc/nodepool/{node_private,sub_nodes_private}); do
+        : $((counter++))
+        echo -e "${ip}\tnode${counter} $(ssh ${ip} hostname)" >> /tmp/hosts
+        echo "node${counter} ansible_connection=${ANSIBLE_CONNECTION_TYPE}" >> ${RAW_INVENTORY}
+    done
+
+    sudo chown root: /tmp/hosts
+    sudo chmod 644 /tmp/hosts
+    sudo mv /tmp/hosts /etc/hosts
+}
+
+function setup_ansible {
+    RAW_INVENTORY=/tmp/kolla/raw_inventory
+    mkdir /tmp/kolla
+
+    sudo -H pip install "ansible<2" "docker-py>=1.6.0"
+
+    setup_inventory
+
+    # Record the running state of the environment as seen by the setup module
+    ansible all -i ${RAW_INVENTORY} -m setup > /tmp/logs/ansible/initial-setup
+}
+
+function setup_node {
+    ansible-playbook -i ${RAW_INVENTORY} tools/setup_nodes.yml
+}
+
+function setup_logging {
+    # This directory is the directory that is copied with the devstack-logs
+    # publisher. It must exist at /home/jenkins/workspace/<job-name>/logs
+    mkdir logs
+
+    # For ease of access we symlink that logs directory to a known path
+    ln -s $(pwd)/logs /tmp/logs
+    mkdir -p /tmp/logs/{ansible,build}
+}
+
+setup_logging
+tools/dump_info.sh
+setup_ssh
+setup_ansible
+setup_node
+setup_config
diff --git a/tools/setup_nodes.yml b/tools/setup_nodes.yml
new file mode 100644
index 000000000..82ca8ae6c
--- /dev/null
+++ b/tools/setup_nodes.yml
@@ -0,0 +1,28 @@
+---
+- hosts: all
+  sudo: yes
+  tasks:
+  - name: Setup /etc/hosts
+    copy:
+      src: /etc/hosts
+      dest: /etc/hosts
+
+  - name: Assign hostname
+    hostname:
+      name: "{{ inventory_hostname }}"
+
+  - name: Copy setup script
+    copy:
+      src: setup_{{ ansible_os_family }}.sh
+      dest: /tmp/setup.sh
+      mode: 0755
+
+- hosts: all
+  tasks:
+  - name: Create log directory for node
+    file:
+      state: directory
+      path: /tmp/{{ inventory_hostname }}
+
+  - name: Run node setup
+    shell: sudo /tmp/setup.sh