Add YAML support to kolla_set_configs engine

Following the PTG agreement, this commit extends the container
configuration engine (set_configs.py) to support YAML format while
maintaining backward compatibility with JSON.

- Integrated 'yaml' library as a universal parser for both JSON
  and YAML formats (YAML is a superset of JSON)
- Updated 'load_config' to search for 'config.json', 'config.yaml'
- Added 'python3-yaml' (Debian) and 'python3-pyyaml' (RHEL) as
  required base packages in Dockerfile.j2

This change improves configuration readability and allows for
easier maintenance while ensuring no breakage for existing
JSON-based deployments.

Change-Id: Icfb52b8f94e1036d5952b160c5921348d1427fda
Signed-off-by: Piotr Milewski <vurmil@gmail.com>

4 files changed, 99 insertions, 8 deletions

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index a4255f98c..3825a2f79 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -138,6 +138,7 @@ RUN {{ macros.install_packages(base_centos_yum_repo_packages | customizable("cen
         'procps-ng',
         'python3',
         'python3-pip',
+        'python3-pyyaml',
         'socat',
         'sudo',
         'tar',
@@ -230,6 +231,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
    'procps',
    'python3',
    'python3-pip',
+   'python3-yaml',
    'socat',
    'sudo',
    'tgt',
diff --git a/docker/base/set_configs.py b/docker/base/set_configs.py
index 1a80bd877..542537e4c 100644
--- a/docker/base/set_configs.py
+++ b/docker/base/set_configs.py
@@ -23,6 +23,7 @@ import re
 import shutil
 import stat
 import sys
+import yaml
 
 
 # TODO(rhallisey): add docstring.
@@ -298,13 +299,33 @@ def validate_source(data):
 
 def load_config():
     def load_from_file():
-        config_file = '/var/lib/kolla/config_files/config.json'
-        LOG.info("Loading config file at %s", config_file)
+        yaml_config_file = '/var/lib/kolla/config_files/config.yaml'
+        json_config_file = '/var/lib/kolla/config_files/config.json'
 
-        # Attempt to read config file
-        with open(config_file) as f:
+        if os.path.exists(yaml_config_file):
+            config_file = yaml_config_file
+            LOG.info("Loading config file at %s", config_file)
             try:
-                return json.load(f)
+                with open(config_file) as f:
+                    config = yaml.safe_load(f)
+
+                    if config is None:
+                        raise InvalidConfig(
+                            "Invalid yaml file found at %s: "
+                            "file is empty" % config_file)
+                    return config
+            except yaml.YAMLError:
+                raise InvalidConfig(
+                    "Invalid yaml file found at %s" % config_file)
+            except IOError as e:
+                raise InvalidConfig(
+                    "Could not read file %s: %r" % (config_file, e))
+        else:
+            config_file = json_config_file
+            LOG.info("Loading config file at %s", config_file)
+            try:
+                with open(config_file) as f:
+                    return json.load(f)
             except ValueError:
                 raise InvalidConfig(
                     "Invalid json file found at %s" % config_file)
@@ -631,7 +652,7 @@ def execute_config_check(config):
     """
     state = get_defaults_state()
 
-    # Build a set of all current destination paths from config.json
+    # Build a set of all current destination paths from the config
     # If the destination is a directory, we append the
     # basename of the source
     current_dests = {
@@ -642,7 +663,7 @@ def execute_config_check(config):
     }
 
     # Detect any paths that are present in the state file but
-    # missing from config.json.
+    # missing from the config file.
     # These would be either restored (if state[dest] has a backup)
     # or removed (if dest is null)
     removed_dests = [
@@ -653,7 +674,7 @@ def execute_config_check(config):
     if removed_dests:
         raise StateMismatch(
             f"The following config files are tracked in state but missing "
-            f"from config.json. "
+            f"from the config file. "
             f"They would be restored or removed: {sorted(removed_dests)}"
         )
 
diff --git a/releasenotes/notes/switch-to-yaml-configs-32802ccb894c0295.yaml b/releasenotes/notes/switch-to-yaml-configs-32802ccb894c0295.yaml
new file mode 100644
index 000000000..bfc798906
--- /dev/null
+++ b/releasenotes/notes/switch-to-yaml-configs-32802ccb894c0295.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    The ``kolla_set_configs`` tool now supports YAML as a native
+    configuration format. It can now load ``config.yaml`` files in
+    addition to the legacy ``config.json``. Existing JSON
+    configurations remain fully supported.
diff --git a/tests/test_set_config.py b/tests/test_set_config.py
index 6e3ddf5e3..89b71fca8 100644
--- a/tests/test_set_config.py
+++ b/tests/test_set_config.py
@@ -75,6 +75,67 @@ class LoadFromFile(base.BaseTestCase):
 
             self.assertEqual(calls, mo.mock_calls)
 
+    @mock.patch('os.path.exists', return_value=True)
+    def test_load_yaml_ok(self, mock_exists):
+        in_config = (
+            'command: /bin/true\n'
+            'config_files: {}\n'
+        )
+
+        mo = mock.mock_open(read_data=in_config)
+        with mock.patch.object(set_configs, 'open', mo):
+            config = set_configs.load_config()
+            self.assertEqual(config['command'], '/bin/true')
+
+        mock_exists.assert_called_with(
+            '/var/lib/kolla/config_files/config.yaml')
+
+    @mock.patch('os.path.exists', return_value=True)
+    def test_load_yaml_invalid(self, mock_exists):
+        in_config = '{'
+
+        mo = mock.mock_open(read_data=in_config)
+        with mock.patch.object(set_configs, 'open', mo):
+            self.assertRaises(set_configs.InvalidConfig,
+                              set_configs.load_config)
+
+    @mock.patch('os.path.exists', return_value=False)
+    def test_load_json_when_no_yaml(self, mock_exists):
+        in_config = json.dumps({'command': '/bin/true',
+                                'config_files': {}})
+
+        mo = mock.mock_open(read_data=in_config)
+        with mock.patch.object(set_configs, 'open', mo):
+            config = set_configs.load_config()
+            self.assertEqual(config['command'], '/bin/true')
+
+        mock_exists.assert_called_with(
+            '/var/lib/kolla/config_files/config.yaml')
+
+    @mock.patch('os.path.exists', return_value=True)
+    def test_load_yaml_full_config(self, mock_exists):
+        in_config = (
+            'command: kolla_toolbox\n'
+            'config_files:\n'
+            '- dest: /etc/rabbitmq/rabbitmq-env.conf\n'
+            '  source: /var/lib/kolla/config_files/rabbitmq-env.conf\n'
+            '  owner: rabbitmq\n'
+            '  perm: \'0600\'\n'
+            'permissions:\n'
+            '- path: /var/log/kolla/ansible.log\n'
+            '  owner: ansible:kolla\n'
+            '  perm: \'0664\'\n'
+        )
+
+        mo = mock.mock_open(read_data=in_config)
+        with mock.patch.object(set_configs, 'open', mo):
+            config = set_configs.load_config()
+
+        self.assertEqual(config['command'], 'kolla_toolbox')
+        self.assertEqual(len(config['config_files']), 1)
+        self.assertEqual(config['config_files'][0]['owner'], 'rabbitmq')
+        self.assertEqual(config['permissions'][0]['perm'], '0664')
+
 
 FAKE_CONFIG_FILES = [
     set_configs.ConfigFile(