From 781d84658ddfdf98307a62830d5a800ad00164ef Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Thu, 6 Apr 2017 13:21:09 +0000 Subject: introduce playbook to ovs with dpdk - This change creates a new role to support deploying ovs with dpdk. - This change introduces an enable_ovs_dpdk variable to enable ovs with dpdk as part of the deploy action. - This change extends the ovs-dpdkctl.sh tool to correctly deploy ovs-dpdk on Ubuntu and CentOS hosts. - This change extends the cleanup-host tool to correctly uninstall ovs-dpdkctl.sh tool and its systemd files if present on the host. - This change automatically configures userspace kernel drivers. partial-implementes: bp/ovs-dpdk Change-Id: I55858d81df437e8258705b75426f61ab9b907c7d --- tools/cleanup-host | 8 ++- tools/kolla-ansible | 2 +- tools/ovs-dpdkctl.sh | 137 ++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 134 insertions(+), 13 deletions(-) (limited to 'tools') diff --git a/tools/cleanup-host b/tools/cleanup-host index 0cf4a69b1..b0917e1e3 100755 --- a/tools/cleanup-host +++ b/tools/cleanup-host @@ -56,8 +56,14 @@ if [[ "$glance_file_datadir_volume" != "glance" && -d "$glance_file_datadir_volu rm -rfv $glance_file_datadir_volume fi -echo "Getting folders name..." FOLDER_PATH="/etc/kolla/" + +if [[ -e "$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh" ]]; then + echo uninstalling ovs-dpdk service files and tools. + sudo CONFIG_FILE="$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.conf" $FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh uninstall +fi + +echo "Getting folders name..." for dir in $FOLDER_PATH*; do if [ "$dir" == "$FOLDER_PATH""passwords.yml" ] || \ [ "$dir" == "$FOLDER_PATH""globals.yml" ] || \ diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 53174cae4..63692267a 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -49,7 +49,7 @@ Commands: prechecks Do pre-deployment checks for hosts check Do post-deployment smoke tests mariadb_recovery Recover a completely stopped mariadb cluster - bootstrap-servers bootstrap servers with kolla deploy dependencies + bootstrap-servers Bootstrap servers with kolla deploy dependencies destroy Destroy Kolla containers, volumes and host configuration --include-images to also destroy Kolla images --include-dev to also destroy dev mode repos diff --git a/tools/ovs-dpdkctl.sh b/tools/ovs-dpdkctl.sh index 2b89ad432..fc5cf2f78 100755 --- a/tools/ovs-dpdkctl.sh +++ b/tools/ovs-dpdkctl.sh @@ -8,6 +8,7 @@ fi FULL_PATH=$(realpath "${BASH_SOURCE[0]}") CONFIG_FILE=${CONFIG_FILE:-"/etc/default/ovs-dpdk.conf"} SERVICE_FILE="/etc/systemd/system/ovs-dpdkctl.service" +BRIDGE_SERVICE_FILE="/etc/systemd/system/ovs-dpdk-bridge.service" function get_value { crudini --get $CONFIG_FILE $@ @@ -37,8 +38,12 @@ function del_config { rm -f $CONFIG_FILE } +function is_redhat_family { + [[ -e /etc/redhat-release ]] ; echo $? +} + function generate_pciwhitelist { - local _Whitelist='' + _Whitelist='' for nic in $(list_dpdk_nics); do address="$(get_value $nic address)" if [ "$_Whitelist" == '' ]; then @@ -79,6 +84,7 @@ function gen_config { set_value ovs ovs_socket_mem ${ovs_socket_mem:-"512"} set_value ovs dpdk_interface_driver ${dpdk_interface_driver:-"uio_pci_generic"} set_value ovs hugepage_mountpoint ${hugepage_mountpoint:-"/dev/hugepages"} + set_value ovs physical_port_policy ${ovs_physical_port_policy:-"named"} ls -al /sys/class/net/* | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print "\n[" a[n] "]\naddress = " a[n-2] "\ndriver ="}' >> $CONFIG_FILE @@ -93,8 +99,8 @@ function gen_config { } function bind_nic { - echo $1 > /sys/bus/pci/drivers/$2/bind echo $2 > /sys/bus/pci/devices/$1/driver_override + echo $1 > /sys/bus/pci/drivers/$2/bind } function unbind_nic { @@ -137,7 +143,6 @@ function unbind_nics { done } - function get_address_by_name { ls -al /sys/class/net/$1 | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print a[n-2] }' } @@ -147,7 +152,7 @@ function get_driver_by_address { } function get_port_bridge { - for pair in $(get_value ovs port port_mappings); do + for pair in $(get_value ovs port_mappings); do nic=`echo $pair | cut -f 1 -d ":"` if [[ "$nic" == "$1" ]]; then bridge=`echo $pair | cut -f 2 -d ":"` @@ -178,11 +183,11 @@ function init_ovs_bridges { } function init_ovs_interfaces { - local pci_port_pairs =='' + pci_port_pairs='' for nic in $(list_dpdk_nics); do address="$(get_value $nic address)" if [ "$pci_port_pairs" == '' ]; then - pci_port_pairs ="$address,$nic" + pci_port_pairs="$address,$nic" else pci_port_pairs="$pci_port_pairs $address,$nic" fi @@ -194,10 +199,13 @@ function init_ovs_interfaces { nic="$(echo $pair | cut -f 2 -d ",")" bridge="$(get_port_bridge $nic)" # ovs 2.6 and older requires dpdkX names, ovs 2.7+ requires dpdk-devargs instead. - ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \ - -- set Interface "dpdk${dpdk_port_number}" type=dpdk || \ - ovs-vsctl --no-wait --may-exist add-port $bridge $nic \ - -- set Interface $nic type=dpdk options:dpdk-devargs=$addr + if [ "$(get_value ovs physical_port_policy)" == "indexed" ]; then + ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \ + -- set Interface "dpdk${dpdk_port_number}" type=dpdk + else + ovs-vsctl --may-exist add-port $bridge $nic \ + -- set Interface $nic type=dpdk options:dpdk-devargs=$addr + fi dpdk_port_number=$((dpdk_port_number+1)) done @@ -210,6 +218,57 @@ function init { init_ovs_interfaces } +function install_network_manager_conf { + pair=$(get_value ovs cidr_mappings) + bridge=`echo $pair | cut -f 1 -d ":"` + cidr=`echo $pair | cut -f 2 -d ":"` + ip=`echo $cidr | cut -f 1 -d "/"` + prefix=`echo $cidr | cut -f 2 -d "/"` + mask="" + full_octets=$(expr $prefix / 8) + partial_octet=$(expr $prefix % 8) + + for octet in 0 1 2 3 ; do + if [[ "$octet" < "$full_octets" ]]; then + mask+=255 + elif [[ "$octet" == "$full_octets" ]]; then + mask+=$((256 - 2**(8-$partial_octet))) + else + mask+=0 + fi + [[ "$octet" < 3 ]] && mask+=. + done + if [[ is_redhat_family == 0 ]]; then + cat << EOF | tee "/etc/sysconfig/network-scripts/ifcfg-$bridge" +DEVICE=$bridge +BOOTPROTO=static +IPADDR=$ip +NETMASK=$mask +HOTPLUG=yes +ONBOOT=yes +EOF +install_redhat_bridge_service $bridge + else + cat << EOF | tee "/etc/network/interfaces.d/$bridge.cfg" + auto $bridge + iface $bridge inet static + address $ip + netmask $mask +EOF + + fi +} + +function uninstall_network_manager_conf { + pair=$(get_value ovs cidr_mappings) + bridge=`echo $pair | cut -f 1 -d ":"` + if [[ is_redhat_family == 0 ]]; then + rm -f /etc/sysconfig/network-scripts/ifcfg-$bridge + else + rm -f /etc/network/interfaces.d/$bridge.cfg + fi +} + function install_service { cat << EOF | tee "$SERVICE_FILE" @@ -235,13 +294,63 @@ EOF systemctl enable ovs-dpdkctl } +function install_redhat_bridge_service { + cat << EOF | tee "$BRIDGE_SERVICE_FILE" +[Unit] +Description=configuration service for ovs-dpdk bridge. +After=docker.service +After=network.target +After=ovs-dpdkctl.service + +[Service] +Type=simple +RemainAfterExit=yes +ExecStartPre=/bin/bash -c "[[ -e /sys/class/net/$1 ]]" +ExecStart=/usr/sbin/ifup $1 +ExecStop=/usr/sbin/ifdown $1 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target + +EOF + systemctl daemon-reload + systemctl enable ovs-dpdk-bridge +} + function uninstall_service { systemctl disable ovs-dpdkctl rm -f "$SERVICE_FILE" + if [ -e "$BRIDGE_SERVICE_FILE" ]; then + systemctl disable ovs-dpdk-bridge + rm -f "$BRIDGE_SERVICE_FILE" + fi systemctl daemon-reload } +function configure_kernel_modules { + driver="$(get_value ovs dpdk_interface_driver)" + lsmod | grep -ws $driver > /dev/null || modprobe $driver + if [[ is_redhat_family == 0 ]]; then + [[ ! -e /etc/modules-load.d/${driver}.conf ]] && echo $driver | tee /etc/modules-load.d/${driver}.conf + else + grep -ws $driver /etc/modules > /dev/null || echo $driver | tee -a /etc/modules + fi +} + +function unconfigure_kernel_modules { + driver="$(get_value ovs dpdk_interface_driver)" + lsmod | grep -ws $driver > /dev/null && rmmod $driver + if [[ is_redhat_family == 0 ]] ; then + [[ -e /etc/modules-load.d/${driver}.conf ]] && rm -f /etc/modules-load.d/${driver}.conf + else + grep -ws $driver /etc/modules > /dev/null && sed -e "s/$driver//" -i /etc/modules + fi +} + function install { + configure_kernel_modules if [ ! -e "$SERVICE_FILE" ]; then install_service fi @@ -253,6 +362,10 @@ function install { gen_config fi systemctl start ovs-dpdkctl + install_network_manager_conf + if [[ is_redhat_family == 0 ]]; then + systemctl start ovs-dpdk-bridge + fi } function uninstall { @@ -260,14 +373,16 @@ function uninstall { if [ -e "$SERVICE_FILE" ]; then uninstall_service fi + uninstall_network_manager_conf + unconfigure_kernel_modules if [ -e /bin/ovs-dpdkctl ]; then rm -f /bin/ovs-dpdkctl fi if [ -e "$CONFIG_FILE" ]; then rm -f "$CONFIG_FILE" fi -} +} function useage { cat << "EOF" -- cgit v1.2.3