From f9ee96f21695f7eba81c07e257eda3a080108b70 Mon Sep 17 00:00:00 2001 From: Sam Yaple Date: Sat, 7 Nov 2015 15:58:10 +0000 Subject: Convert gate to Ansible setup To support multinode we must now distribute our setup to multiple hosts. Instead of making special rules for this, we are going to convert our existing setup to Ansible. This way both setup proceedures take place in the exact same fashion. Partially-Implements: blueprint multinode-gate Change-Id: I43ece298bba994e9b5083403ef3cf6d4245cda6d --- tests/setup_Debian.sh | 39 +++++++++++++++++++ tests/setup_RedHat.sh | 45 ++++++++++++++++++++++ tests/setup_centos.sh | 40 ------------------- tests/setup_deploy.sh | 31 --------------- tests/setup_gate.sh | 105 ++++++++++++++++++++++++++++++++++++++++++-------- tests/setup_nodes.yml | 28 ++++++++++++++ tests/setup_ubuntu.sh | 36 ----------------- tox.ini | 3 -- 8 files changed, 201 insertions(+), 126 deletions(-) create mode 100755 tests/setup_Debian.sh create mode 100755 tests/setup_RedHat.sh delete mode 100755 tests/setup_centos.sh delete mode 100755 tests/setup_deploy.sh create mode 100644 tests/setup_nodes.yml delete mode 100755 tests/setup_ubuntu.sh diff --git a/tests/setup_Debian.sh b/tests/setup_Debian.sh new file mode 100755 index 000000000..c9159e003 --- /dev/null +++ b/tests/setup_Debian.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +DEV=$1 + +# (SamYaple)TODO: Remove the path overriding +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +# Setup Docker repo and add signing key +echo 'deb http://apt.dockerproject.org/repo ubuntu-trusty main' | sudo tee /etc/apt/sources.list.d/docker.list +sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D +sudo apt-get update +sudo apt-get install -y --no-install-recommends docker-engine=1.8.2-0~trusty btrfs-tools + +# The reason for using BTRFS is stability. There are numerous issues with the +# devicemapper backed on Ubuntu and AUFS is slow. BTRFS is very solid as a +# backend in my experince. I use ie almost exclusively. +# Format Disks and setup Docker to use BTRFS +sudo umount /dev/${DEV} || true +sudo parted /dev/${DEV} -s -- mklabel msdos +sudo service docker stop +echo 'DOCKER_OPTS="-s btrfs"' | sudo tee /etc/default/docker +sudo rm -rf /var/lib/docker/* + +# We want to snapshot the entire docker directory so we have to first create a +# subvolume and use that as the root for the docker directory. +sudo mkfs.btrfs -f /dev/${DEV} +sudo mount /dev/${DEV} /var/lib/docker +sudo btrfs subvolume create /var/lib/docker/docker +sudo umount /var/lib/docker +sudo mount -o noatime,compress=lzo,space_cache,subvol=docker /dev/${DEV} /var/lib/docker + +sudo service docker start + +sudo docker info + +echo "Completed $0." diff --git a/tests/setup_RedHat.sh b/tests/setup_RedHat.sh new file mode 100755 index 000000000..e9920a0da --- /dev/null +++ b/tests/setup_RedHat.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +DEV=$1 + +# (SamYaple)TODO: Remove the path overriding +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +cat | sudo tee /etc/yum.repos.d/docker.repo << EOF +[docker] +name=Docker Main Repository +baseurl=https://yum.dockerproject.org/repo/main/centos/7 +enabled=1 +gpgcheck=1 +gpgkey=https://yum.dockerproject.org/gpg +EOF + +sudo yum install -y libffi-devel openssl-devel docker-engine-1.8.2 xfsprogs + +# Setup backing disk for use with Docker. This is to ensure we use the ephemeral +# disk provided to the build instance. It ensures the correct disk and storage +# driver are used for Docker. It is recommend to use the thin provisioning +# driver. https://github.com/docker/docker/blob/master/man/docker.1.md +sudo parted /dev/${DEV} -s -- mklabel msdos mkpart pri 1 -1 +# Figure out the path to the partitioned device +PARTDEV=$(ls "/dev/${DEV}"* | egrep "/dev/${DEV}p?1") +sudo pvcreate ${PARTDEV} +sudo vgcreate kolla01 ${PARTDEV} +sudo lvcreate -n thin01 -L 60G kolla01 +sudo lvcreate -n thin01meta -L 2G kolla01 +yes | sudo lvconvert --type thin-pool --poolmetadata kolla01/thin01meta kolla01/thin01 + +# Setup Docker +sudo sed -i -r 's,(ExecStart)=(.+),\1=/usr/bin/docker daemon --storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=kolla01-thin01 --storage-opt dm.use_deferred_removal=true,' /usr/lib/systemd/system/docker.service +sudo systemctl daemon-reload +sudo systemctl start docker +sudo docker info + +# disable ipv6 until we're sure routes to fedora mirrors work properly +sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf' +sudo /usr/sbin/sysctl -p + +echo "Completed $0." diff --git a/tests/setup_centos.sh b/tests/setup_centos.sh deleted file mode 100755 index 5800cce3c..000000000 --- a/tests/setup_centos.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -cat | sudo tee /etc/yum.repos.d/docker.repo << EOF -[docker] -name=Docker Main Repository -baseurl=https://yum.dockerproject.org/repo/main/centos/7 -enabled=1 -gpgcheck=1 -gpgkey=https://yum.dockerproject.org/gpg -EOF - -sudo yum install -y libffi-devel openssl-devel docker-engine-1.8.2 xfsprogs - -# Setup backing disk for use with Docker. This is to ensure we use the ephemeral -# disk provided to the build instance. It ensures the correct disk and storage -# driver are used for Docker. It is recommend to use the thin provisioning -# driver. https://github.com/docker/docker/blob/master/man/docker.1.md -sudo parted /dev/${DEV} -s -- mklabel msdos mkpart pri 1 -1 -# Figure out the path to the partitioned device -PARTDEV=$(ls "/dev/${DEV}"* | egrep "/dev/${DEV}p?1") -sudo pvcreate ${PARTDEV} -sudo vgcreate kolla01 ${PARTDEV} -sudo lvcreate -n thin01 -L 60G kolla01 -sudo lvcreate -n thin01meta -L 2G kolla01 -yes | sudo lvconvert --type thin-pool --poolmetadata kolla01/thin01meta kolla01/thin01 - -# Setup Docker -sudo sed -i -r 's,(ExecStart)=(.+),\1=/usr/bin/docker daemon --storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=kolla01-thin01 --storage-opt dm.use_deferred_removal=true,' /usr/lib/systemd/system/docker.service -sudo systemctl daemon-reload -sudo systemctl start docker -sudo docker info - -# disable ipv6 until we're sure routes to fedora mirrors work properly -sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf' -sudo /usr/sbin/sysctl -p - -echo "Completed $0." diff --git a/tests/setup_deploy.sh b/tests/setup_deploy.sh deleted file mode 100755 index 366bbf5a8..000000000 --- a/tests/setup_deploy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - -function create_keys { - # Setup ssh key as required - sudo -H ssh-keygen -f /root/.ssh/id_rsa -N "" - sudo -H cat /root/.ssh/id_rsa.pub | sudo -H tee /root/.ssh/authorized_keys -} - -function install_deps { - # Install Ansible and docker-py - sudo -H pip install "ansible<2" docker-py - pip freeze | egrep "docker|ansible" -} - -function copy_configs { - # Copy configs - sudo cp -a etc/kolla /etc/ -} - -create_keys -install_deps -copy_configs - -# Link the logs directory into root -mkdir -p logs -sudo ln -s $(pwd)/logs /root/logs diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh index 7919f4ae9..b270c03c2 100755 --- a/tests/setup_gate.sh +++ b/tests/setup_gate.sh @@ -3,25 +3,98 @@ set -o xtrace set -o errexit -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - -# TODO(SamYaple): This check could be much better, but should work for now -if [[ $(hostname | grep rax) ]]; then - export DEV="xvde" -else - echo "Assuming this is an hpcloud box" - export DEV="vdb" -fi - # Just for mandre :) if [[ ! -f /etc/sudoers.d/jenkins ]]; then echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins fi -# This creates a logs directory for the devstack logs publisher -mkdir logs -# We symlink the logs dir to a known location -ln -s $(pwd)/logs /tmp/logs +function detect_disk { + # TODO(SamYaple): This check could be much better, but should work for now + if [[ $(hostname | grep rax) ]]; then + export DEV="xvde" + else + echo "Assuming this is an hpcloud box" + export DEV="vdb" + fi +} + +function setup_config { + # Copy configs + sudo cp -a etc/kolla /etc/ +} + +function detect_distro { + DISTRO=$(ansible all -i "localhost," -msetup -clocal | awk -F\" '/ansible_os_family/ {print $4}') +} + +function setup_ssh { + # Generate a new keypair that Ansible will use + ssh-keygen -f /home/jenkins/.ssh/kolla -N '' + cat /home/jenkins/.ssh/kolla.pub >> /home/jenkins/.ssh/authorized_keys + + # Push the the public key around to all of the nodes + for ip in $(cat /etc/nodepool/sub_nodes_private); do + scp /home/jenkins/.ssh/kolla.pub ${ip}:/home/jenkins/.ssh/authorized_keys + # TODO(SamYaple): Remove this root key pushing once Kolla doesn't + # require root anymore. + ssh ${ip} -i /home/jenkins/.ssh/kolla 'sudo mkdir -p /root/.ssh; sudo cp /home/jenkins/.ssh/* /root/.ssh/' + done + + # From now on use the new IdentityFile for connecting to other hosts + echo "IdentityFile /home/jenkins/.ssh/kolla" >> /home/jenkins/.ssh/config +} + +function setup_inventory { + local counter=0 + + detect_distro + if [[ "${DISTRO}" == "Debian" ]]; then + ANSIBLE_CONNECTION_TYPE=ssh + else + ANSIBLE_CONNECTION_TYPE=local + fi + + echo -e "127.0.0.1\tlocalhost" > /tmp/hosts + for ip in $(cat /etc/nodepool/{node_private,sub_nodes_private}); do + : $((counter++)) + echo -e "${ip}\tnode${counter} $(ssh ${ip} hostname)" >> /tmp/hosts + echo "node${counter} ansible_connection=${ANSIBLE_CONNECTION_TYPE}" >> ${RAW_INVENTORY} + done + + sudo chown root: /tmp/hosts + sudo chmod 644 /tmp/hosts + sudo mv /tmp/hosts /etc/hosts +} + +function setup_ansible { + RAW_INVENTORY=/tmp/kolla/raw_inventory + mkdir /tmp/kolla + + sudo -H pip install "ansible<2" docker-py + + setup_inventory + + # Record the running state of the environment as seen by the setup module + ansible all -i ${RAW_INVENTORY} -m setup > /tmp/logs/ansible/initial-setup +} + +function setup_node { + detect_disk + ansible-playbook -i ${RAW_INVENTORY} -edocker_dev=${DEV} tests/setup_nodes.yml +} + +function setup_logging { + # This directory is the directory that is copied with the devstack-logs + # publisher. It must exist at /home/jenkins/workspace//logs + mkdir logs + + # For ease of access we symlink that logs directory to a known path + ln -s $(pwd)/logs /tmp/logs + mkdir -p /tmp/logs/{ansible,build} +} -distro=$(awk -F'[="]'+ '/^ID/ {print tolower($2); exit}' /etc/*-release) -exec tests/setup_${distro}.sh +setup_logging +setup_ssh +setup_ansible +setup_node +setup_config diff --git a/tests/setup_nodes.yml b/tests/setup_nodes.yml new file mode 100644 index 000000000..e23da173d --- /dev/null +++ b/tests/setup_nodes.yml @@ -0,0 +1,28 @@ +--- +- hosts: all + sudo: yes + tasks: + - name: Setup /etc/hosts + copy: + src: /etc/hosts + dest: /etc/hosts + + - name: Assign hostname + hostname: + name: "{{ inventory_hostname }}" + + - name: Copy setup script + copy: + src: setup_{{ ansible_os_family }}.sh + dest: /tmp/setup.sh + mode: 0755 + +- hosts: all + tasks: + - name: Create log directory for node + file: + state: directory + path: /tmp/{{ inventory_hostname }} + + - name: Run node setup + shell: sudo /tmp/setup.sh {{ docker_dev }} diff --git a/tests/setup_ubuntu.sh b/tests/setup_ubuntu.sh deleted file mode 100755 index 66280da34..000000000 --- a/tests/setup_ubuntu.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -sudo mount - -# Setup Docker repo and add signing key -echo 'deb http://apt.dockerproject.org/repo ubuntu-trusty main' | sudo tee /etc/apt/sources.list.d/docker.list -sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -sudo apt-get update -sudo apt-get install -y --no-install-recommends docker-engine=1.8.2-0~trusty btrfs-tools - -# The reason for using BTRFS is stability. There are numerous issues with the -# devicemapper backed on Ubuntu and AUFS is slow. BTRFS is very solid as a -# backend in my experince. I use ie almost exclusively. -# Format Disks and setup Docker to use BTRFS -sudo umount /dev/${DEV} || true -sudo parted /dev/${DEV} -s -- mklabel msdos -sudo service docker stop -echo 'DOCKER_OPTS="-s btrfs"' | sudo tee /etc/default/docker -sudo rm -rf /var/lib/docker/* - -# We want to snapshot the entire docker directory so we have to first create a -# subvolume and use that as the root for the docker directory. -sudo mkfs.btrfs -f /dev/${DEV} -sudo mount /dev/${DEV} /var/lib/docker -sudo btrfs subvolume create /var/lib/docker/docker -sudo umount /var/lib/docker -sudo mount -o noatime,compress=lzo,space_cache,subvol=docker /dev/${DEV} /var/lib/docker - -sudo service docker start - -sudo docker info - -echo "Completed $0." diff --git a/tox.ini b/tox.ini index 3ae566149..c7909a2ab 100644 --- a/tox.ini +++ b/tox.ini @@ -70,7 +70,6 @@ commands = find . -type f -name "*.pyc" -delete bash -c "if [ ! -d .testrepository ]; then testr init; fi" sudo -g docker testr run test_build.DeployTestCentosBinary - bash tests/setup_deploy.sh sudo tests/deploy_aio.sh centos binary [testenv:deploy-centos-source] @@ -81,7 +80,6 @@ commands = find . -type f -name "*.pyc" -delete bash -c "if [ ! -d .testrepository ]; then testr init; fi" sudo -g docker testr run test_build.DeployTestCentosSource - bash tests/setup_deploy.sh sudo tests/deploy_aio.sh centos source [testenv:deploy-ubuntu-source] @@ -92,7 +90,6 @@ commands = find . -type f -name "*.pyc" -delete bash -c "if [ ! -d .testrepository ]; then testr init; fi" sudo -g docker testr run test_build.DeployTestUbuntuSource - bash tests/setup_deploy.sh sudo tests/deploy_aio.sh ubuntu source [testenv:deploy-multinode-ubuntu-source] -- cgit v1.2.3