Migrate to pyproject.toml

[tool.setuptools.data-files] does not support recursive directory
installation. The old setup.cfg [files] worked because PBR's
FilesConfig.expand_globs() used os.walk() to expand docker/*, but
this hook is not triggered by the pbr.build PEP-517 backend.

Move docker/ into kolla/ and use [tool.setuptools.package-data]
instead. Update kolla_worker._get_images_dir() to check the kolla
package directory first so both editable and regular installs work.

Change-Id: I4fc452c03740ef22ebbb4e8d0d86c4f4783e2d09
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>

10 files changed, 0 insertions, 429 deletions

diff --git a/docker/keystone/keystone-base/Dockerfile.j2 b/docker/keystone/keystone-base/Dockerfile.j2
deleted file mode 100644
index faa424fa7..000000000
--- a/docker/keystone/keystone-base/Dockerfile.j2
+++ /dev/null
@@ -1,45 +0,0 @@
-FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }}
-{% block labels %}
-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}"
-{% endblock %}
-
-{% block keystone_base_header %}{% endblock %}
-
-{% import "macros.j2" as macros with context %}
-
-{{ macros.configure_user(name='keystone') }}
-
-{% if base_package_type == 'rpm' %}
-
-{{ macros.enable_extra_repos(['epel']) }}
-
-    {% set keystone_base_packages = [
-        'cyrus-sasl-devel',
-        'openldap-devel',
-    ] %}
-
-{% elif base_package_type == 'deb' %}
-    {% set keystone_base_packages = [
-        'libldap-common',
-        'libldap2-dev',
-        'libsasl2-dev',
-    ] %}
-{% endif %}
-{{ macros.install_packages(keystone_base_packages | customizable("packages")) }}
-
-{% block keystone_source_install %}
-ADD keystone-base-archive /keystone-base-source
-
-{% set keystone_base_pip_packages = [
-    '/keystone[ldap]',
-] %}
-
-RUN ln -s keystone-base-source/* keystone \
-    && {{ macros.install_pip(keystone_base_pip_packages | customizable("pip_packages")) }} \
-    && mkdir -p /etc/keystone \
-    && cp -r /keystone/etc/* /etc/keystone/
-{% endblock %}
-
-{{ macros.kolla_patch_sources() }}
-
-{% block keystone_base_footer %}{% endblock %}
diff --git a/docker/keystone/keystone-fernet/Dockerfile.j2 b/docker/keystone/keystone-fernet/Dockerfile.j2
deleted file mode 100644
index da32ddb86..000000000
--- a/docker/keystone/keystone-fernet/Dockerfile.j2
+++ /dev/null
@@ -1,34 +0,0 @@
-FROM {{ namespace }}/{{ image_prefix }}keystone-base:{{ tag }}
-{% block labels %}
-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}"
-{% endblock %}
-
-{% block keystone_fernet_header %}{% endblock %}
-
-{% import "macros.j2" as macros with context %}
-
-{% if base_package_type == 'rpm' %}
-    {% set keystone_fernet_packages = [
-        'cronie',
-        'openssh-clients',
-        'rsync'
-    ] %}
-{% elif base_package_type == 'deb' %}
-    {% set keystone_fernet_packages = [
-        'cron',
-        'openssh-client',
-        'rsync'
-    ] %}
-{% endif %}
-{{ macros.install_packages(keystone_fernet_packages | customizable("packages")) }}
-
-COPY fetch_fernet_tokens.py /usr/bin/
-COPY keystone_bootstrap.sh /usr/local/bin/kolla_keystone_bootstrap
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-RUN chmod 644 /usr/local/bin/kolla_extend_start \
-    && chmod 755 /usr/local/bin/kolla_keystone_bootstrap /usr/bin/fetch_fernet_tokens.py
-
-{{ macros.kolla_patch_sources() }}
-
-{% block keystone_fernet_footer %}{% endblock %}
-{% block footer %}{% endblock %}
diff --git a/docker/keystone/keystone-fernet/extend_start.sh b/docker/keystone/keystone-fernet/extend_start.sh
deleted file mode 100644
index cf356949f..000000000
--- a/docker/keystone/keystone-fernet/extend_start.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-FERNET_SYNC=/usr/bin/fernet-node-sync.sh
-FERNET_TOKEN_DIR="/etc/keystone/fernet-keys"
-
-if [[ -f "${FERNET_SYNC}" ]]; then
-    ${FERNET_SYNC}
-fi
-
-if [[ $(stat -c %U:%G ${FERNET_TOKEN_DIR}) != "keystone:keystone" ]]; then
-    chown keystone:keystone ${FERNET_TOKEN_DIR}
-fi
diff --git a/docker/keystone/keystone-fernet/fetch_fernet_tokens.py b/docker/keystone/keystone-fernet/fetch_fernet_tokens.py
deleted file mode 100644
index c57943630..000000000
--- a/docker/keystone/keystone-fernet/fetch_fernet_tokens.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/python
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Basically this module will fetch the fernet tokens and compare them to the
-# required time constrains to determine whether the host needs to resync with
-# other nodes in the cluster.
-import argparse
-from datetime import datetime
-from datetime import timedelta
-import json
-import os
-import sys
-
-# Adding nosec since this fails bandit B105, 'Possible hardcoded password'.
-TOKEN_PATH = '/etc/keystone/fernet-keys'  # nosec
-
-
-def json_exit(msg=None, failed=False, changed=False):
-    if type(msg) is not dict:
-        msg = {'msg': str(msg)}
-    msg.update({'failed': failed, 'changed': changed})
-    print(json.dumps(msg))
-    sys.exit()
-
-
-def has_file(filename_path):
-    if not os.path.exists(filename_path):
-        return False
-    return True
-
-
-def num_tokens():
-    _, _, files = next(os.walk(TOKEN_PATH))
-    return len(files)
-
-
-def tokens_populated(expected):
-    return num_tokens() >= int(expected)
-
-
-def token_stale(seconds, filename='0'):
-    max_token_age = datetime.now() - timedelta(seconds=int(seconds))
-    filename_path = os.path.join(TOKEN_PATH, filename)
-
-    if not has_file(filename_path):
-        return True
-    modified_date = datetime.fromtimestamp(os.path.getmtime(filename_path))
-    return modified_date < max_token_age
-
-
-def main():
-    parser = argparse.ArgumentParser(description='''Checks to see if a fernet
-        token no older than a desired time.''')
-    parser.add_argument('-t', '--time',
-                        help='Time in seconds for a token rotation',
-                        required=True)
-    parser.add_argument('-f', '--filename',
-                        help='Filename of token to check',
-                        default='0')
-    parser.add_argument('-n', '--number',
-                        help='Minimum number of tokens that should exist',
-                        required=True)
-    args = parser.parse_args()
-
-    json_exit({
-        'populated': tokens_populated(args.number),
-        'update_required': token_stale(args.time, args.filename),
-    })
-
-
-if __name__ == '__main__':
-    main()
diff --git a/docker/keystone/keystone-fernet/keystone_bootstrap.sh b/docker/keystone/keystone-fernet/keystone_bootstrap.sh
deleted file mode 100644
index d361767bb..000000000
--- a/docker/keystone/keystone-fernet/keystone_bootstrap.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -x
-
-USERNAME=$1
-GROUP=$2
-
-function fail_json {
-    echo '{"failed": true, "msg": "'$1'", "changed": true}'
-    exit 1
-}
-
-function exit_json {
-    echo '{"failed": false, "changed": '"${changed}"'}'
-}
-
-changed="false"
-keystone_bootstrap=$(keystone-manage --config-file /etc/keystone/keystone.conf fernet_setup --keystone-user ${USERNAME} --keystone-group ${GROUP} 2>&1)
-if [[ $? != 0 ]]; then
-    fail_json "${keystone_bootstrap}"
-fi
-
-changed=$(echo "${keystone_bootstrap}" | awk '
-    /Key repository is already initialized/ {count++}
-    END {
-        if (count == 1) changed="true"; else changed="false"
-        print changed
-    }'
-)
-
-exit_json
diff --git a/docker/keystone/keystone-ssh/Dockerfile.j2 b/docker/keystone/keystone-ssh/Dockerfile.j2
deleted file mode 100644
index af290bd5f..000000000
--- a/docker/keystone/keystone-ssh/Dockerfile.j2
+++ /dev/null
@@ -1,38 +0,0 @@
-FROM {{ namespace }}/{{ image_prefix }}keystone-base:{{ tag }}
-{% block labels %}
-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}"
-{% endblock %}
-
-{% block keystone_ssh_header %}{% endblock %}
-
-{% import "macros.j2" as macros with context %}
-
-{{ macros.configure_user(name='keystone', shell='/bin/bash') }}
-
-{% if base_package_type == 'rpm' %}
-    {% set keystone_ssh_packages = [
-        'openssh-server',
-        'rsync'
-    ] %}
-
-{% elif base_package_type == 'deb' %}
-    {% set keystone_ssh_packages = [
-        'openssh-server',
-        'rsync'
-    ] %}
-
-RUN mkdir -p /var/run/sshd \
-    && chmod 0755 /var/run/sshd
-
-{% endif %}
-{{ macros.install_packages(keystone_ssh_packages | customizable("packages")) }}
-
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-
-RUN sed -ri 's/session(\s+)required(\s+)pam_loginuid.so/session\1optional\2pam_loginuid.so/' /etc/pam.d/sshd \
-    && chmod 644 /usr/local/bin/kolla_extend_start
-
-{{ macros.kolla_patch_sources() }}
-
-{% block keystone_ssh_footer %}{% endblock %}
-{% block footer %}{% endblock %}
diff --git a/docker/keystone/keystone-ssh/extend_start.sh b/docker/keystone/keystone-ssh/extend_start.sh
deleted file mode 100644
index 559cd65a9..000000000
--- a/docker/keystone/keystone-ssh/extend_start.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# NOTE(mnasiadka): CentOS 10 does not support dsa
-SSH_HOST_KEY_TYPES=( "rsa" "ecdsa" "ed25519" )
-
-for key_type in ${SSH_HOST_KEY_TYPES[@]}; do
-    KEY_PATH=/etc/ssh/ssh_host_${key_type}_key
-    if [[ ! -f "${KEY_PATH}" ]]; then
-        ssh-keygen -q -t ${key_type} -f ${KEY_PATH} -N ""
-    fi
-done
-
-mkdir -p /var/lib/keystone/.ssh
-
-if [[ $(stat -c %U:%G /var/lib/keystone/.ssh) != "keystone:keystone" ]]; then
-    sudo chown keystone: /var/lib/keystone/.ssh
-fi
diff --git a/docker/keystone/keystone/Dockerfile.j2 b/docker/keystone/keystone/Dockerfile.j2
deleted file mode 100644
index 742626c05..000000000
--- a/docker/keystone/keystone/Dockerfile.j2
+++ /dev/null
@@ -1,45 +0,0 @@
-FROM {{ namespace }}/{{ image_prefix }}keystone-base:{{ tag }}
-{% block labels %}
-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}"
-{% endblock %}
-
-{% block keystone_header %}{% endblock %}
-
-{% import "macros.j2" as macros with context %}
-
-{% if base_package_type == 'rpm' %}
-    {% set keystone_packages = [
-        'krb5-devel',
-    ] %}
-{% elif base_package_type == 'deb' %}
-    {% set keystone_packages = [
-        'libkrb5-dev',
-    ] %}
-{% endif %}
-{{ macros.install_packages(keystone_packages | customizable("packages")) }}
-
-{% set keystone_pip_packages = [
-    'requests-kerberos',
-] %}
-
-RUN {{ macros.install_pip(keystone_pip_packages | customizable("pip_packages")) }}
-
-ADD plugins-archive /
-
-{% set keystone_plugins_pip_packages = [
-    '/plugins/*',
-] %}
-
-COPY keystone_bootstrap.sh /usr/local/bin/kolla_keystone_bootstrap
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-
-RUN chmod 644 /usr/local/bin/kolla_extend_start \
-    && chmod 755 /usr/local/bin/kolla_keystone_bootstrap \
-    && if [ "$(ls /plugins)" ]; then \
-        {{ macros.install_pip(keystone_plugins_pip_packages) }}; \
-    fi
-
-{{ macros.kolla_patch_sources() }}
-
-{% block keystone_footer %}{% endblock %}
-{% block footer %}{% endblock %}
diff --git a/docker/keystone/keystone/extend_start.sh b/docker/keystone/keystone/extend_start.sh
deleted file mode 100644
index b45c94d0d..000000000
--- a/docker/keystone/keystone/extend_start.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-# Create log dir for Keystone logs
-KEYSTONE_LOG_DIR="/var/log/kolla/keystone"
-if [[ ! -d "${KEYSTONE_LOG_DIR}" ]]; then
-    mkdir -p ${KEYSTONE_LOG_DIR}
-fi
-if [[ $(stat -c %U:%G ${KEYSTONE_LOG_DIR}) != "keystone:kolla" ]]; then
-    chown keystone:kolla ${KEYSTONE_LOG_DIR}
-fi
-if [ ! -f "${KEYSTONE_LOG_DIR}/keystone.log" ]; then
-    touch ${KEYSTONE_LOG_DIR}/keystone.log
-fi
-if [[ $(stat -c %U:%G ${KEYSTONE_LOG_DIR}/keystone.log) != "keystone:keystone" ]]; then
-    chown keystone:keystone ${KEYSTONE_LOG_DIR}/keystone.log
-fi
-if [[ $(stat -c %a ${KEYSTONE_LOG_DIR}) != "755" ]]; then
-    chmod 755 ${KEYSTONE_LOG_DIR}
-fi
-
-EXTRA_KEYSTONE_MANAGE_ARGS=${EXTRA_KEYSTONE_MANAGE_ARGS-}
-# Upgrade and exit if KOLLA_UPGRADE variable is set. This catches all cases
-# of the KOLLA_UPGRADE variable being set, including empty.
-if [[ "${!KOLLA_UPGRADE[@]}" ]]; then
-    # TODO(duonghq): check doctor result here
-    # TODO: find reason why doctor failed in gate
-    # sudo -H -u keystone keystone-manage doctor
-    sudo -H -u keystone keystone-manage ${EXTRA_KEYSTONE_MANAGE_ARGS} db_sync --expand
-    sudo -H -u keystone keystone-manage ${EXTRA_KEYSTONE_MANAGE_ARGS} db_sync --migrate
-    exit 0
-fi
-
-# Contract database and exit if KOLLA_FINISH_UPGRADE variable is set.
-# This catches all cases of the KOLLA_FINISH_UPGRADE variable being set,
-# including empty.
-if [[ "${!KOLLA_FINISH_UPGRADE[@]}" ]]; then
-    sudo -H -u keystone keystone-manage ${EXTRA_KEYSTONE_MANAGE_ARGS} db_sync --contract
-    exit 0
-fi
-
-# Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
-# of the KOLLA_BOOTSTRAP variable being set, including empty.
-if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    sudo -H -u keystone keystone-manage ${EXTRA_KEYSTONE_MANAGE_ARGS} db_sync
-    exit 0
-fi
diff --git a/docker/keystone/keystone/keystone_bootstrap.sh b/docker/keystone/keystone/keystone_bootstrap.sh
deleted file mode 100644
index 1b04d99b6..000000000
--- a/docker/keystone/keystone/keystone_bootstrap.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/bash
-
-set -o pipefail
-
-# NOTE(SamYaple): Kolla needs to wraps `keystone-manage bootstrap` to ensure
-# any change is reported correctly for idempotency. This script will exit with
-# valid json that can be parsed with information about if the task has failed
-# and if anything changed.
-
-USERNAME=$1
-PASSWORD=$2
-if [ -z "$PASSWORD" ]; then
-    # Avoid having the password always come in via CLI (which makes
-    # it show up in things like ara)
-    PASSWORD="$OS_BOOTSTRAP_PASSWORD"
-fi
-PROJECT=$3
-ROLE=$4
-if [ $# -ge 8 ]; then
-    # ignored ADMIN_URL compat mode
-    # ADMIN_URL=$5
-    INTERNAL_URL=$6
-    PUBLIC_URL=$7
-    REGION=$8
-else
-    # no ADMIN_URL modern mode
-    INTERNAL_URL=$5
-    PUBLIC_URL=$6
-    REGION=$7
-fi
-
-function fail_json {
-    echo '{"failed": true, "msg": "'$1'", "changed": true}'
-    exit 1
-}
-
-function exit_json {
-    echo '{"failed": false, "changed": '"${changed}"'}'
-}
-
-changed="false"
-# NOTE(mgoddard): pipe through cat -v to remove unprintable control characters
-# which prevent JSON decoding.
-# NOTE(yoctozepto): also apply sed to escape double quotation marks
-# and backslashes
-keystone_bootstrap=$(keystone-manage bootstrap --bootstrap-username "${USERNAME}" --bootstrap-password "${PASSWORD}" --bootstrap-project-name "${PROJECT}" --bootstrap-role-name "${ROLE}" --bootstrap-internal-url "${INTERNAL_URL}" --bootstrap-public-url "${PUBLIC_URL}" --bootstrap-service-name "keystone" --bootstrap-region-id "${REGION}" 2>&1 | cat -v | sed 's/\\/\\\\/g' | sed 's/"/\\"/g')
-if [[ $? != 0 ]]; then
-    fail_json "${keystone_bootstrap}"
-fi
-
-changed=$(echo "${keystone_bootstrap}" | awk '
-    /Domain default already exists, skipping creation./ ||
-    /Project '"${PROJECT}"' already exists, skipping creation./ ||
-    /User '"${USERNAME}"' already exists, skipping creation./ ||
-    /Role '"${ROLE}"' exists, skipping creation./ ||
-    /User '"${USERNAME}"' already has '"${ROLE}"' on '"${PROJECT}"'./ ||
-    /Region '"${REGION}"' exists, skipping creation./ ||
-    /Skipping internal endpoint as already created/ ||
-    /Skipping public endpoint as already created/ {count++}
-    END {
-        if (count == 9) changed="false"; else changed="true"
-        print changed
-    }'
-)
-
-exit_json