diff options
| author | Michal Nasiadka <mnasiadka@gmail.com> | 2026-07-06 15:26:00 +0200 |
|---|---|---|
| committer | Michal Nasiadka <mnasiadka@gmail.com> | 2026-07-06 19:59:40 +0000 |
| commit | db35ac8d793506bc43b62e7e39d27a75de1043ed (patch) | |
| tree | feec28d13a43490f5fd4260374144a1abfc20331 /docker/base/set_configs.py | |
| parent | 1ae1850557e3b99902f73b32c9005618537a1f2d (diff) | |
| download | kolla-db35ac8d793506bc43b62e7e39d27a75de1043ed.tar.gz kolla-db35ac8d793506bc43b62e7e39d27a75de1043ed.zip | |
Migrate to pyproject.toml
[tool.setuptools.data-files] does not support recursive directory
installation. The old setup.cfg [files] worked because PBR's
FilesConfig.expand_globs() used os.walk() to expand docker/*, but
this hook is not triggered by the pbr.build PEP-517 backend.
Move docker/ into kolla/ and use [tool.setuptools.package-data]
instead. Update kolla_worker._get_images_dir() to check the kolla
package directory first so both editable and regular installs work.
Change-Id: I4fc452c03740ef22ebbb4e8d0d86c4f4783e2d09
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>
Diffstat (limited to 'docker/base/set_configs.py')
| -rw-r--r-- | docker/base/set_configs.py | 759 |
1 files changed, 0 insertions, 759 deletions
diff --git a/docker/base/set_configs.py b/docker/base/set_configs.py deleted file mode 100644 index 4b025d55a..000000000 --- a/docker/base/set_configs.py +++ /dev/null @@ -1,759 +0,0 @@ -#!/usr/bin/env python3 - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import glob -import grp -import json -import logging -import os -import pwd -import re -import shutil -import stat -import sys -import yaml - - -# TODO(rhallisey): add docstring. -logging.basicConfig( - format='%(asctime)s.%(msecs)03d %(levelname)s %(message)s', - level=logging.INFO, - datefmt='%Y-%m-%d %H:%M:%S' -) -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.INFO) - -KOLLA_DEFAULTS = "/etc/kolla/defaults" -KOLLA_DEFAULTS_STATE = KOLLA_DEFAULTS + '/' + 'state' - - -class ExitingException(Exception): - def __init__(self, message, exit_code=1): - super(ExitingException, self).__init__(message) - self.exit_code = exit_code - - -class ImmutableConfig(ExitingException): - pass - - -class InvalidConfig(ExitingException): - pass - - -class MissingRequiredSource(ExitingException): - pass - - -class UserNotFound(ExitingException): - pass - - -class ConfigFileBadState(ExitingException): - pass - - -class ConfigFileCommandDiffers(ExitingException): - pass - - -class StateMismatch(ExitingException): - pass - - -class ConfigFile(object): - - def __init__(self, source, dest, owner=None, perm=None, optional=False, - preserve_properties=False, merge=False): - self.source = source - self.dest = dest - self.owner = owner - self.perm = perm - self.optional = optional - self.merge = merge - self.preserve_properties = preserve_properties - - def __str__(self): - return '<ConfigFile source:"{}" dest:"{}">'.format(self.source, - self.dest) - - def _copy_file(self, source, dest): - self._delete_path(dest) - # dest endswith / means copy the <source> to <dest> folder - LOG.info('Copying %s to %s', source, dest) - if self.merge and self.preserve_properties and os.path.islink(source): - link_target = os.readlink(source) - os.symlink(link_target, dest) - else: - shutil.copy(source, dest) - self._set_properties(source, dest) - - def _merge_directories(self, source, dest): - if os.path.isdir(source): - if os.path.lexists(dest) and not os.path.isdir(dest): - self._delete_path(dest) - if not os.path.isdir(dest): - LOG.info('Creating directory %s', dest) - os.makedirs(dest) - self._set_properties(source, dest) - - dir_content = os.listdir(source) - for to_copy in dir_content: - self._merge_directories(os.path.join(source, to_copy), - os.path.join(dest, to_copy)) - else: - self._copy_file(source, dest) - - def _delete_path(self, path): - if not os.path.lexists(path): - return - LOG.info('Deleting %s', path) - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.remove(path) - - def _create_parent_dirs(self, path): - parent_path = os.path.dirname(path) - if not os.path.exists(parent_path): - os.makedirs(parent_path) - - def _set_properties(self, source, dest): - if self.preserve_properties: - self._set_properties_from_file(source, dest) - else: - self._set_properties_from_conf(dest) - - def _set_properties_from_file(self, source, dest): - LOG.info('Copying permissions from %s onto %s', source, dest) - shutil.copystat(source, dest) - stat = os.stat(source) - os.chown(dest, stat.st_uid, stat.st_gid) - - def _set_properties_from_conf(self, path): - config = {'permissions': - [{'owner': self.owner, 'path': path, 'perm': self.perm}]} - handle_permissions(config) - - def copy(self): - - sources = glob.glob(self.source) - - if not self.optional and not sources: - raise MissingRequiredSource('%s file is not found' % self.source) - # skip when there is no sources and optional - elif self.optional and not sources: - return - - for source in sources: - dest = self.dest - # dest endswith / means copy the <source> into <dest> folder, - # otherwise means copy the source to dest - if dest.endswith(os.sep): - dest = os.path.join(dest, os.path.basename(source)) - if not self.merge: - self._delete_path(dest) - self._create_parent_dirs(dest) - try: - self._merge_directories(source, dest) - except OSError: - # If a source is tried to merge with a read-only mount, it - # may throw an OSError. Because we don't print the source or - # dest anywhere, let's catch the exception and log a better - # message to help with tracking down the issue. - LOG.error('Unable to merge %s with %s', source, dest) - raise - - def _cmp_file(self, source, dest): - # check exist - if (os.path.exists(source) and - not os.path.exists(dest)): - return False - if (os.path.exists(dest) and - not os.path.exists(source)): - return False - # check content - with open(source, 'rb') as f1, open(dest, 'rb') as f2: - if f1.read() != f2.read(): - LOG.error('The content of source file(%s) and' - ' dest file(%s) are not equal.', source, dest) - return False - # check perm - file_stat = os.stat(dest) - actual_perm = oct(file_stat.st_mode)[-4:] - if self.perm != actual_perm: - LOG.error('Dest file does not have expected perm: %s, actual: %s', - self.perm, actual_perm) - return False - # check owner - desired_user, desired_group = user_group(self.owner) - actual_user = pwd.getpwuid(file_stat.st_uid) - if actual_user.pw_name != desired_user: - LOG.error('Dest file does not have expected user: %s,' - ' actual: %s ', desired_user, actual_user.pw_name) - return False - actual_group = grp.getgrgid(file_stat.st_gid) - if actual_group.gr_name != desired_group: - LOG.error('Dest file does not have expected group: %s,' - ' actual: %s ', desired_group, actual_group.gr_name) - return False - return True - - def _cmp_dir(self, source, dest): - for root, dirs, files in os.walk(source): - for dir_ in dirs: - full_path = os.path.join(root, dir_) - dest_full_path = os.path.join(dest, os.path.relpath(source, - full_path)) - dir_stat = os.stat(dest_full_path) - actual_perm = oct(dir_stat.st_mode)[-4:] - if self.perm != actual_perm: - LOG.error('Dest dir does not have expected perm: %s,' - ' actual %s', self.perm, actual_perm) - return False - for file_ in files: - full_path = os.path.join(root, file_) - dest_full_path = os.path.join(dest, os.path.relpath(full_path, - source)) - if not self._cmp_file(full_path, dest_full_path): - return False - return True - - def check(self): - bad_state_files = [] - sources = glob.glob(self.source) - - if not sources and not self.optional: - raise MissingRequiredSource('%s file is not found' % self.source) - elif self.optional and not sources: - return - - for source in sources: - dest = self.dest - # dest endswith / means copy the <source> into <dest> folder, - # otherwise means copy the source to dest - if dest.endswith(os.sep): - dest = os.path.join(dest, os.path.basename(source)) - if os.path.isdir(source): - if not self._cmp_dir(source, dest): - bad_state_files.append(source) - elif not self._cmp_file(source, dest): - bad_state_files.append(source) - if len(bad_state_files) != 0: - msg = 'Following files are in bad state: %s' % bad_state_files - raise ConfigFileBadState(msg) - - -def validate_config(config): - required_keys = {'source', 'dest'} - required_path_keys = {'path', 'owner', 'perm'} - - if 'command' not in config: - raise InvalidConfig('Config is missing required "command" key') - - # Validate config sections - for data in config.get('config_files', list()): - # Verify required keys exist. - if not set(data.keys()) >= required_keys: - message = 'Config is missing required keys: %s' % required_keys - raise InvalidConfig(message) - if ('owner' not in data or 'perm' not in data) \ - and not data.get('preserve_properties', False): - raise InvalidConfig( - 'Config needs preserve_properties or owner and perm') - - for section in ('directories', 'files'): - for data in config.get(section, list()): - missing = required_path_keys - set(data.keys()) - if missing: - raise InvalidConfig( - '%s config is missing required keys: %s' - % (section, missing)) - - for key in required_path_keys: - if data.get(key) is None: - raise InvalidConfig( - '%s config key "%s" must not be null' - % (section, key)) - - if glob.has_magic(data['path']): - raise InvalidConfig( - '%s config does not support glob paths: %s' - % (section, data['path'])) - - -def validate_source(data): - source = data.get('source') - - # Only check existence if no wildcard found - if '*' not in source: - if not os.path.exists(source): - if data.get('optional'): - LOG.info("%s does not exist, but is not required", source) - return False - else: - raise MissingRequiredSource( - "The source to copy does not exist: %s" % source) - - return True - - -def load_config(): - def load_from_file(): - yaml_config_file = '/var/lib/kolla/config_files/config.yaml' - json_config_file = '/var/lib/kolla/config_files/config.json' - - if os.path.exists(yaml_config_file): - config_file = yaml_config_file - LOG.info("Loading config file at %s", config_file) - try: - with open(config_file) as f: - config = yaml.safe_load(f) - - if config is None: - raise InvalidConfig( - "Invalid yaml file found at %s: " - "file is empty" % config_file) - return config - except yaml.YAMLError: - raise InvalidConfig( - "Invalid yaml file found at %s" % config_file) - except IOError as e: - raise InvalidConfig( - "Could not read file %s: %r" % (config_file, e)) - else: - config_file = json_config_file - LOG.info("Loading config file at %s", config_file) - try: - with open(config_file) as f: - return json.load(f) - except ValueError: - raise InvalidConfig( - "Invalid json file found at %s" % config_file) - except IOError as e: - raise InvalidConfig( - "Could not read file %s: %r" % (config_file, e)) - - config = load_from_file() - - LOG.info('Validating config file') - validate_config(config) - return config - - -def copy_config(config): - if 'config_files' in config: - LOG.info('Copying service configuration files') - for data in config['config_files']: - config_file = ConfigFile(**data) - config_file.copy() - else: - LOG.debug('No files to copy found in config') - - LOG.info('Writing out command to execute') - LOG.debug("Command is: %s", config['command']) - # The value from the 'command' key will be written to '/run_command' - cmd = '/run_command' - with open(cmd, 'w+') as f: - f.write(config['command']) - # Make sure the generated file is readable by all users - try: - os.chmod(cmd, 0o644) - except OSError: - LOG.exception('Failed to set permission of %s to 0o644', cmd) - - -def create_directories(config): - if 'directories' not in config: - return - LOG.info('Creating directories') - for data in config['directories']: - path = data['path'] - try: - os.makedirs(path, exist_ok=True) - except Exception: - raise OSError("Can't create destination directory (%s)" % (path)) - handle_permissions({'permissions': [data]}) - - -def create_files(config): - if 'files' not in config: - return - LOG.info('Creating files') - for data in config['files']: - path = data['path'] - if os.path.islink(path): - raise OSError( - "Path is a symbolic link, expected regular file (%s)" % path) - if os.path.isdir(path): - raise OSError( - "Path is a directory, expected file (%s)" % path) - parent = os.path.dirname(path) - if parent and not os.path.isdir(parent): - raise OSError( - "Parent directory does not exist (%s)" % parent) - if not os.path.exists(path): - with open(path, 'a'): - pass - handle_permissions({'permissions': [data]}) - - -def user_group(owner): - if ':' in owner: - user, group = owner.split(':', 1) - if not group: - group = user - else: - user, group = owner, owner - return user, group - - -def handle_permissions(config): - for permission in config.get('permissions', list()): - path = permission.get('path') - owner = permission.get('owner') - recurse = permission.get('recurse', False) - perm = permission.get('perm') - exclude = permission.get('exclude', []) - - desired_user, desired_group = user_group(owner) - uid = pwd.getpwnam(desired_user).pw_uid - gid = grp.getgrnam(desired_group).gr_gid - - def set_perms(path, uid, gid, perm): - LOG.info('Setting permission for %s', path) - if not os.path.exists(path): - LOG.warning('%s does not exist', path) - return - - try: - os.chown(path, uid, gid) - except OSError: - LOG.exception('Failed to change ownership of %s to %s:%s', - path, uid, gid) - - if perm: - # Accept valid UNIX permission notation - # (e.g. 644, 0600, 1777, 2755) - # or equivalent Python octal literals - # (e.g. 0o644, 0o1777, 0o2755). - if re.fullmatch(r'[0-7]{3,4}', perm): - perm = int(perm, 8) - elif re.fullmatch(r'0o[0-7]{3,4}', perm): - perm = int(perm, 0) - else: - raise ValueError( - f"Invalid permission '{perm}'. Expected a valid UNIX " - "permission (e.g. 644, 0600, 2755) or a Python octal " - "literal (e.g. 0o644, 0o0600, 0o2755)." - ) - - # Ensure execute bit on directory if read bit is set - if os.path.isdir(path): - if perm & stat.S_IRUSR: - perm |= stat.S_IXUSR - if perm & stat.S_IRGRP: - perm |= stat.S_IXGRP - if perm & stat.S_IROTH: - perm |= stat.S_IXOTH - - try: - os.chmod(path, perm) - except OSError: - LOG.exception('Failed to set permission of %s to %s', - path, perm) - - def handle_exclusion(root, path_suffix): - full_path = os.path.join(root, path_suffix) - LOG.debug("Checking for exclusion: %s" % full_path) - if exclude: - for exclude_ in exclude: - if not re.search(exclude_, full_path): - set_perms(full_path, uid, gid, perm) - else: - set_perms(full_path, uid, gid, perm) - - for dest in glob.glob(path): - set_perms(dest, uid, gid, perm) - if recurse and os.path.isdir(dest): - for root, dirs, files in os.walk(dest): - for dir_ in dirs: - handle_exclusion(root, dir_) - for file_ in files: - handle_exclusion(root, file_) - - -def get_defaults_state(): - """Retrieve the saved default configuration state from default state file. - - This function creates the directory for Kolla defaults if it does not - exist, and then attempts to read the current configuration state from - a JSON file. If the file exists, it reads and returns the content. - If not, it returns an empty dictionary. - - Simply said, when the container starts for the first time, the state file - doesn't exist, and it returns an empty dictionary. - However, if it has already been started before, it will contain the state - as it was when it first ran. - - Returns: - dict: The configuration state stored in the Kolla defaults state file. - - Example: - { - "/etc/cinder/cinder.conf": { - "source": "/etc/cinder/cinder.conf", - "preserve_properties": true, - "dest": null - }, - "/etc/apache2/conf-enabled/cinder-wsgi.conf": { - "source": "/etc/apache2/conf-enabled/cinder-wsgi.conf", - "preserve_properties": true, - "dest": null - }, - "/etc/cinder/cinder_audit_map.conf": { - "source": "/etc/cinder/cinder_audit_map.conf", - "preserve_properties": true, - "dest": "/etc/kolla/defaults/etc/cinder/cinder_audit_map.conf" - } - } - - From above example: - /etc/cinder/cinder.conf didn't exist - /etc/apache2/conf-enabled/cinder-wsgi.conf didn't exist - /etc/cinder/cinder_audit_map.conf exists and saved - """ - os.makedirs(KOLLA_DEFAULTS, exist_ok=True) - if os.path.exists(KOLLA_DEFAULTS_STATE): - with open(KOLLA_DEFAULTS_STATE, 'r') as f: - return json.load(f) - else: - return {} - - -def set_defaults_state(state): - """Save the provided configuration state to the defaults state file. - - This function writes the provided state (a dictionary) to a JSON file at - the specified Kolla defaults state location, ensuring that it is properly - formatted with indentation for readability. - - Args: - state (dict): The configuration state to save to the Kolla defaults - state file. - """ - with open(KOLLA_DEFAULTS_STATE, 'w') as f: - json.dump(state, f, indent=4) - - -def remove_or_restore_configs(state): - """Remove or restore configuration files based on their current state. - - This function iterates over the configuration files in the provided state. - If the destination is `None`, it removes the file or directory. Otherwise, - it swaps the source and destination, restoring the configuration file - by copying it back to its original location. - - Args: - state (dict): The current default state of configuration files, mapping - file paths to their source and destination information. - """ - for k, v in state.items(): - if v['dest'] is None: - if os.path.exists(k): - if os.path.isfile(k): - os.remove(k) - else: - shutil.rmtree(k) - else: - v['source'], v['dest'] = v['dest'], v['source'] - config_file = ConfigFile(**v) - config_file.copy() - - -def backup_configs(config, state): - """Back up new configuration files and update the default state. - - This function processes new configuration files provided in the - input `config`. For each file, it checks if the destination exists in the - current state. If not, it backs up the file by copying it to the default - directory. It then updates the state with the new configuration file's - information. - - Args: - config (dict): The input configuration containing a list of config - files. - state (dict): The current default state to be updated with the new - config files. - """ - if 'config_files' in config: - for data in config['config_files']: - if data['dest'] in state.keys(): - continue - src = data['source'] - if data['dest'].endswith('/'): - dst = data['dest'] + data['source'].split('/')[-1] - else: - dst = data['dest'] - default = KOLLA_DEFAULTS + dst - if os.path.exists(src): - copy = {'source': dst, 'preserve_properties': True} - if os.path.exists(dst): - copy['dest'] = default - if dst not in state: - config_file = ConfigFile(**copy) - config_file.copy() - state[dst] = copy - else: - copy['dest'] = None - if dst not in state: - state[dst] = copy - - -def handle_defaults(config): - """Handle the default config files by copying/removing them as needed. - - This function loads the current default state and manages the configuration - files. It first processes existing configuration files in the default - state, either removing or restoring them based on their destination status. - It then backs up any new configuration files from the input config, - updating the default state accordingly. - - Args: - config (dict): A dictionary containing the list of configuration files - to be handled. - """ - state = get_defaults_state() - remove_or_restore_configs(state) - backup_configs(config, state) - set_defaults_state(state) - - -def execute_config_strategy(config): - config_strategy = os.environ.get("KOLLA_CONFIG_STRATEGY") - LOG.info("Kolla config strategy set to: %s", config_strategy) - if config_strategy == "COPY_ALWAYS": - create_directories(config) - create_files(config) - handle_defaults(config) - copy_config(config) - handle_permissions(config) - elif config_strategy == "COPY_ONCE": - if os.path.exists('/configured'): - raise ImmutableConfig( - "The config strategy prevents copying new configs", - exit_code=0) - else: - create_directories(config) - create_files(config) - handle_defaults(config) - copy_config(config) - handle_permissions(config) - os.mknod('/configured') - else: - raise InvalidConfig('KOLLA_CONFIG_STRATEGY is not set properly') - - -def execute_command_check(config): - cmd = config.get('command') - with open("/run_command", "r") as f: - cmd_running = f.read() - if cmd != cmd_running: - msg = "Running command differs. " + cmd + " != " + cmd_running - raise ConfigFileCommandDiffers(msg) - - -def execute_config_check(config): - """Check configuration state consistency and validate config file entries. - - This function compares the current config file destinations from the - provided config dictionary with those stored in the defaults state file. - If any destinations are found in the state file but not in the config, - a StateMismatch exception is raised. These missing files would otherwise - be restored or removed depending on their backup state. - - After validating consistency, the function performs standard checks on - each declared configuration file, including content, permissions, and - ownership validation. - - Args: - config (dict): The configuration dictionary containing 'config_files' - entries as expected by Kolla. - - Raises: - StateMismatch: If there are entries in the defaults state not present - in the provided config. - """ - state = get_defaults_state() - - # Build a set of all current destination paths from the config - # If the destination is a directory, we append the - # basename of the source - current_dests = { - entry['dest'] if not entry['dest'].endswith('/') else - os.path.join(entry['dest'], os.path.basename(entry['source'])) - for entry in config.get('config_files', []) - if entry.get('dest') - } - - # Detect any paths that are present in the state file but - # missing from the config file. - # These would be either restored (if state[dest] has a backup) - # or removed (if dest is null) - removed_dests = [ - path for path in state.keys() - if path not in current_dests - ] - - if removed_dests: - raise StateMismatch( - f"The following config files are tracked in state but missing " - f"from the config file. " - f"They would be restored or removed: {sorted(removed_dests)}" - ) - - # Perform the regular content, permissions, and ownership - # checks on the declared files - for data in config.get('config_files', []): - config_file = ConfigFile(**data) - config_file.check() - - -def main(): - try: - parser = argparse.ArgumentParser() - parser.add_argument('--check', - action='store_true', - required=False, - help='Check whether the configs changed') - args = parser.parse_args() - config = load_config() - - if args.check: - execute_command_check(config) - execute_config_check(config) - else: - execute_config_strategy(config) - except ExitingException as e: - LOG.error("%s: %s", e.__class__.__name__, e) - return e.exit_code - except Exception: - LOG.exception('Unexpected error:') - return 2 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) |
