kolla-toolbox: Template clouds.yaml and use it

Basically Ansible heuristic obfuscation methods mark auth:
that we pass to openstack.* modules as no_log_values and then
obfuscates it's values in other runs (which then end up as ****
e.g. auth_url, domain_id, domain name, etc)

Example return:

    "domains": [
        {
            "description": "The ******** domain",
            "id": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
            "is_enabled": true,
            "links": {
                "self": "********/v3/domains/********"
            },
            "name": "Default",
            "options": {},
            "tags": []
        }

This patch adds clouds.yaml to files templated for kolla-toolbox
(but without the password) and only uses password from Kolla-Ansible
secrets instead of storing that on disk on all of the nodes.

Change-Id: I479445563d5b5dc2e45601e936d89def647da841
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>

23 files changed, 225 insertions, 98 deletions

diff --git a/ansible/group_vars/all/common.yml b/ansible/group_vars/all/common.yml
index 1aeae7f44..d007b9b1a 100644
--- a/ansible/group_vars/all/common.yml
+++ b/ansible/group_vars/all/common.yml
@@ -298,15 +298,11 @@ om_rabbitmq_qos_prefetch_count: "1"
 
 om_enable_rabbitmq_stream_fanout: true
 
-# OpenStack authentication string. You should only need to override these if you
-# are changing the admin tenant/project or user.
 openstack_auth:
-  auth_url: "{{ keystone_internal_url }}"
-  username: "{{ keystone_admin_user }}"
   password: "{{ keystone_admin_password }}"
-  project_name: "{{ keystone_admin_project }}"
-  domain_name: "default"
-  user_domain_name: "default"
+
+openstack_auth_cloud: "kolla-admin-internal"
+openstack_auth_cloud_system: "kolla-admin-system-internal"
 
 ####################
 # OpenStack options
diff --git a/ansible/group_vars/all/octavia.yml b/ansible/group_vars/all/octavia.yml
index eaff670bc..f95a49722 100644
--- a/ansible/group_vars/all/octavia.yml
+++ b/ansible/group_vars/all/octavia.yml
@@ -31,3 +31,6 @@ octavia_api_port: "9876"
 octavia_api_listen_port: "{{ octavia_api_port }}"
 octavia_api_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else octavia_api_port }}"
 octavia_health_manager_port: "5555"
+# Project that Octavia will use to interact with other services.  Note that in
+# Train and earlier releases this was "admin".
+octavia_service_auth_project: "service"
diff --git a/ansible/roles/blazar/tasks/bootstrap.yml b/ansible/roles/blazar/tasks/bootstrap.yml
index d84eb6eac..88cbbf201 100644
--- a/ansible/roles/blazar/tasks/bootstrap.yml
+++ b/ansible/roles/blazar/tasks/bootstrap.yml
@@ -43,7 +43,8 @@
     container_engine: "{{ kolla_container_engine }}"
     module_name: os_nova_host_aggregate
     module_args:
-      auth: "{{ openstack_auth }}"
+      auth: "{{ openstack_blazar_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
diff --git a/ansible/roles/common/templates/kolla-toolbox.json.j2 b/ansible/roles/common/templates/kolla-toolbox.json.j2
new file mode 100644
index 000000000..bc766febe
--- /dev/null
+++ b/ansible/roles/common/templates/kolla-toolbox.json.j2
@@ -0,0 +1,42 @@
+{
+    "command": "kolla_toolbox",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/clouds.yaml",
+            "dest": "/var/lib/ansible/.config/openstack/clouds.yaml",
+            "owner": "ansible",
+            "perm": "0600"
+        }{% if enable_rabbitmq | bool %},
+        {
+            "source": "{{ container_config_directory }}/rabbitmq-erlang.cookie",
+            "dest": "/var/lib/rabbitmq/.erlang.cookie",
+            "owner": "rabbitmq",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/rabbitmq-env.conf",
+            "dest": "/etc/rabbitmq/rabbitmq-env.conf",
+            "owner": "rabbitmq",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/erl_inetrc",
+            "dest": "/etc/rabbitmq/erl_inetrc",
+            "owner": "rabbitmq",
+            "perm": "0600"
+        }{% endif %}{% if kolla_copy_ca_into_containers | bool %},
+        {
+            "source": "{{ container_config_directory }}/ca-certificates",
+            "dest": "/var/lib/kolla/share/ca-certificates",
+            "owner": "root",
+            "perm": "0600"
+        }{% endif %}
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/ansible.log",
+            "perm": "0664",
+            "owner": "ansible:kolla"
+        }
+    ]
+}
diff --git a/ansible/roles/heat/defaults/main.yml b/ansible/roles/heat/defaults/main.yml
index ec82d57c0..bc22c1a35 100644
--- a/ansible/roles/heat/defaults/main.yml
+++ b/ansible/roles/heat/defaults/main.yml
@@ -260,8 +260,8 @@ heat_ks_roles:
   - "{{ heat_stack_user_role }}"
 
 heat_ks_user_roles:
-  - project: "{{ openstack_auth.project_name }}"
-    user: "{{ openstack_auth.username }}"
+  - project: "{{ keystone_admin_project }}"
+    user: "{{ keystone_admin_user }}"
     role: "{{ heat_stack_owner_role }}"
 
 ####################
diff --git a/ansible/roles/heat/tasks/bootstrap_service.yml b/ansible/roles/heat/tasks/bootstrap_service.yml
index 38fd5d777..70a3ac402 100644
--- a/ansible/roles/heat/tasks/bootstrap_service.yml
+++ b/ansible/roles/heat/tasks/bootstrap_service.yml
@@ -10,13 +10,13 @@
     environment:
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      OS_AUTH_URL: "{{ openstack_auth.auth_url }}"
+      OS_AUTH_URL: "{{ keystone_internal_url }}"
       OS_IDENTITY_API_VERSION: "3"
       OS_INTERFACE: "internal"
-      OS_USERNAME: "{{ openstack_auth.username }}"
-      OS_PASSWORD: "{{ openstack_auth.password }}"
-      OS_PROJECT_NAME: "{{ openstack_auth.project_name }}"
-      OS_USER_DOMAIN_NAME: "{{ openstack_auth.user_domain_name }}"
+      OS_USERNAME: "{{ keystone_admin_user }}"
+      OS_PASSWORD: "{{ keystone_admin_password }}"
+      OS_PROJECT_NAME: "{{ keystone_admin_project }}"
+      OS_USER_DOMAIN_NAME: "{{ default_user_domain_name }}"
       OS_REGION_NAME: "{{ openstack_region_name }}"
       OS_CACERT: "{{ openstack_cacert | default(omit) }}"
       HEAT_DOMAIN_ADMIN_PASSWORD: "{{ heat_domain_admin_password }}"
diff --git a/ansible/roles/ironic/tasks/upgrade.yml b/ansible/roles/ironic/tasks/upgrade.yml
index 7303686ad..8af59c2ee 100644
--- a/ansible/roles/ironic/tasks/upgrade.yml
+++ b/ansible/roles/ironic/tasks/upgrade.yml
@@ -52,19 +52,13 @@
 
 - name: Wait for Ironic nodes not to wait
   become: true
-  vars:
-    ironic_auth:
-      auth_url: "{{ openstack_auth.auth_url }}"
-      username: "{{ openstack_auth.username }}"
-      password: "{{ openstack_auth.password }}"
-      user_domain_name: "{{ openstack_auth.user_domain_name }}"
-      system_scope: "all"
   kolla_toolbox:
     container_engine: "{{ kolla_container_engine }}"
     module_name: "openstack.cloud.baremetal_node_info"
     module_args:
       region_name: "{{ openstack_region_name }}"
-      auth: "{{ ironic_auth }}"
+      auth: "{{ openstack_auth }}"
+      cloud: "{{ openstack_auth_cloud_system }}"
       interface: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
   register: ironic_nodes
diff --git a/ansible/roles/keystone/tasks/register.yml b/ansible/roles/keystone/tasks/register.yml
index c5d730b94..cea5330c5 100644
--- a/ansible/roles/keystone/tasks/register.yml
+++ b/ansible/roles/keystone/tasks/register.yml
@@ -3,13 +3,15 @@
   become: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec keystone kolla_keystone_bootstrap
-    {{ openstack_auth.username }} {{ openstack_auth.password }} {{ openstack_auth.project_name }}
-    admin {{ keystone_internal_url }} {{ keystone_public_url }} {{ item }}
+    {{ keystone_admin_user }} {{ keystone_admin_password }} {{ keystone_admin_project }}
+    admin {{ keystone_internal_url }} {{ keystone_public_url }} {{ region }}
   register: keystone_bootstrap
   changed_when: (keystone_bootstrap.stdout | from_json).changed
   failed_when: (keystone_bootstrap.stdout | from_json).failed
   run_once: true
-  with_items: "{{ multiple_regions_names }}"
+  loop: "{{ multiple_regions_names }}"
+  loop_control:
+    loop_var: region
 
 - name: Register keystone service, endpoints, and users in Keystone
   ansible.builtin.import_role:
@@ -27,6 +29,7 @@
     module_args:
       name: "{{ keystone_default_user_role }}"
       auth: "{{ openstack_keystone_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       endpoint_type: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
       region_name: "{{ openstack_region_name }}"
diff --git a/ansible/roles/keystone/tasks/register_identity_providers.yml b/ansible/roles/keystone/tasks/register_identity_providers.yml
index 475b8f38a..07ee8e436 100644
--- a/ansible/roles/keystone/tasks/register_identity_providers.yml
+++ b/ansible/roles/keystone/tasks/register_identity_providers.yml
@@ -2,13 +2,13 @@
 - name: List configured attribute mappings (that can be used by IdPs)
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-      --os-auth-url={{ openstack_auth.auth_url }}
-      --os-password={{ openstack_auth.password }}
-      --os-username={{ openstack_auth.username }}
+      --os-auth-url={{ keystone_internal_url }}
+      --os-password={{ keystone_admin_password }}
+      --os-username={{ keystone_admin_user }}
       --os-identity-api-version=3
       --os-interface={{ openstack_interface }}
       --os-system-scope="all"
-      --os-user-domain-name={{ openstack_auth.user_domain_name }}
+      --os-user-domain-name={{ default_user_domain_name }}
       --os-region-name={{ openstack_region_name }}
       {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }} {% endif %}
     mapping list -c ID --format value
@@ -25,12 +25,12 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-system-scope="all"
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }} {% endif %}
@@ -51,6 +51,7 @@
     module_args:
       name: "{{ item.openstack_domain }}"
       auth: "{{ openstack_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       endpoint_type: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
       region_name: "{{ openstack_region_name }}"
@@ -62,13 +63,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface {{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }} {% endif %}
     mapping create
@@ -84,13 +85,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }} {% endif %}
     mapping set
@@ -105,13 +106,13 @@
   become: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }} {% endif %}
     identity provider list -c ID --format value
@@ -128,13 +129,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }}{% endif %}
     identity provider delete {{ item }}
@@ -149,13 +150,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }}{% endif %}
     identity provider create
@@ -173,13 +174,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface {{ openstack_interface }}
     --os-system-scope "all"
-    --os-user-domain-name {{ openstack_auth.user_domain_name }}
+    --os-user-domain-name {{ default_user_domain_name }}
     --os-region-name {{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert {{ openstack_cacert }} {% endif %}
     identity provider set
@@ -196,13 +197,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }}{% endif %}
     federation protocol create
@@ -219,13 +220,13 @@
   changed_when: true
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec -t keystone openstack
-    --os-auth-url={{ openstack_auth.auth_url }}
-    --os-password={{ openstack_auth.password }}
-    --os-username={{ openstack_auth.username }}
+    --os-auth-url={{ keystone_internal_url }}
+    --os-password={{ keystone_admin_password }}
+    --os-username={{ keystone_admin_user }}
     --os-identity-api-version=3
     --os-interface={{ openstack_interface }}
     --os-system-scope="all"
-    --os-user-domain-name={{ openstack_auth.user_domain_name }}
+    --os-user-domain-name={{ default_user_domain_name }}
     --os-region-name={{ openstack_region_name }}
     {% if openstack_cacert != '' %}--os-cacert={{ openstack_cacert }}{% endif %}
     federation protocol set
diff --git a/ansible/roles/kolla_toolbox/tasks/config.yml b/ansible/roles/kolla_toolbox/tasks/config.yml
index 899799a94..2672f9b08 100644
--- a/ansible/roles/kolla_toolbox/tasks/config.yml
+++ b/ansible/roles/kolla_toolbox/tasks/config.yml
@@ -72,3 +72,12 @@
   when:
     - kolla_toolbox_services['kolla-toolbox'] | service_enabled_and_mapped_to_host
     - enable_rabbitmq | bool
+
+- name: Copy clouds.yaml to kolla toolbox
+  ansible.builtin.template:
+    src: "clouds.yaml.j2"
+    dest: "{{ node_config_directory }}/kolla-toolbox/clouds.yaml"
+    mode: "0660"
+  become: true
+  when:
+    - kolla_toolbox_services['kolla-toolbox'] | service_enabled_and_mapped_to_host
diff --git a/ansible/roles/kolla_toolbox/templates/clouds.yaml.j2 b/ansible/roles/kolla_toolbox/templates/clouds.yaml.j2
new file mode 100644
index 000000000..1cacc7d55
--- /dev/null
+++ b/ansible/roles/kolla_toolbox/templates/clouds.yaml.j2
@@ -0,0 +1,57 @@
+clouds:
+  kolla-admin:
+    auth:
+      auth_url: {{ keystone_public_url }}
+      project_domain_name: {{ default_project_domain_name }}
+      user_domain_name: {{ default_user_domain_name }}
+      project_name: {{ keystone_admin_project }}
+      username: {{ keystone_admin_user }}
+    region_name: {{ openstack_region_name }}
+{% if kolla_admin_openrc_cacert is not none and kolla_admin_openrc_cacert | length > 0 %}
+    cacert: {{ kolla_admin_openrc_cacert }}
+{% endif %}
+  kolla-admin-system:
+    auth:
+      auth_url: {{ keystone_public_url }}
+      user_domain_name: {{ default_user_domain_name }}
+      system_scope: all
+      username: {{ keystone_admin_user }}
+    region_name: {{ openstack_region_name }}
+{% if kolla_admin_openrc_cacert is not none and kolla_admin_openrc_cacert | length > 0 %}
+    cacert: {{ kolla_admin_openrc_cacert }}
+{% endif %}
+  kolla-admin-internal:
+    auth:
+      auth_url: {{ keystone_internal_url }}
+      project_domain_name: {{ default_project_domain_name }}
+      user_domain_name: {{ default_user_domain_name }}
+      project_name: {{ keystone_admin_project }}
+      username: {{ keystone_admin_user }}
+    interface: internal
+    region_name: {{ openstack_region_name }}
+{% if kolla_admin_openrc_cacert is not none and kolla_admin_openrc_cacert | length > 0 %}
+    cacert: {{ kolla_admin_openrc_cacert }}
+{% endif %}
+  kolla-admin-system-internal:
+    auth:
+      auth_url: {{ keystone_internal_url }}
+      user_domain_name: {{ default_user_domain_name }}
+      system_scope: all
+      username: {{ keystone_admin_user }}
+    interface: internal
+    region_name: {{ openstack_region_name }}
+{% if kolla_admin_openrc_cacert is not none and kolla_admin_openrc_cacert | length > 0 %}
+    cacert: {{ kolla_admin_openrc_cacert }}
+{% endif %}
+  kolla-octavia-internal:
+    auth:
+      auth_url: {{ keystone_internal_url }}
+      project_domain_name: {{ default_project_domain_name }}
+      user_domain_name: {{ default_user_domain_name }}
+      project_name: {{ octavia_service_auth_project }}
+      username: octavia
+    interface: internal
+    region_name: {{ openstack_region_name }}
+{% if kolla_admin_openrc_cacert is not none and kolla_admin_openrc_cacert | length > 0 %}
+    cacert: {{ kolla_admin_openrc_cacert }}
+{% endif %}
diff --git a/ansible/roles/kolla_toolbox/templates/kolla-toolbox.json.j2 b/ansible/roles/kolla_toolbox/templates/kolla-toolbox.json.j2
index 03c79a5cd..712a4782e 100644
--- a/ansible/roles/kolla_toolbox/templates/kolla-toolbox.json.j2
+++ b/ansible/roles/kolla_toolbox/templates/kolla-toolbox.json.j2
@@ -1,7 +1,13 @@
 {
     "command": "kolla_toolbox",
     "config_files": [
-        {% if enable_rabbitmq | bool %}{
+        {
+            "source": "{{ container_config_directory }}/clouds.yaml",
+            "dest": "/var/lib/ansible/.config/openstack/clouds.yaml",
+            "owner": "ansible",
+            "perm": "0600"
+        }{% if enable_rabbitmq | bool %},
+        {
             "source": "{{ container_config_directory }}/rabbitmq-erlang.cookie",
             "dest": "/var/lib/rabbitmq/.erlang.cookie",
             "owner": "rabbitmq",
@@ -18,7 +24,13 @@
             "dest": "/etc/rabbitmq/erl_inetrc",
             "owner": "rabbitmq",
             "perm": "0600"
-        }{% endif %}{% if kolla_copy_ca_into_containers | bool %}{% if enable_rabbitmq | bool %},{% endif %}
+        },
+        {
+            "source": "{{ container_config_directory }}/clouds.yaml",
+            "dest": "/var/lib/ansible/.config/openstack/clouds.yaml",
+            "owner": "ansible",
+            "perm": "0600"
+        }{% endif %}{% if kolla_copy_ca_into_containers | bool %},
         {
             "source": "{{ container_config_directory }}/ca-certificates",
             "dest": "/var/lib/kolla/share/ca-certificates",
diff --git a/ansible/roles/magnum/tasks/register.yml b/ansible/roles/magnum/tasks/register.yml
index b36c7d762..dfe99761a 100644
--- a/ansible/roles/magnum/tasks/register.yml
+++ b/ansible/roles/magnum/tasks/register.yml
@@ -16,6 +16,7 @@
       name: "{{ magnum_trustee_domain }}"
       description: "Owns users and projects created by magnum"
       auth: "{{ openstack_magnum_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       endpoint_type: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
       region_name: "{{ openstack_region_name }}"
@@ -33,6 +34,7 @@
       password: "{{ magnum_keystone_password }}"
       update_password: "{{ 'always' if update_keystone_service_user_passwords | bool else 'on_create' }}"
       auth: "{{ openstack_magnum_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       endpoint_type: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
       region_name: "{{ openstack_region_name }}"
@@ -48,6 +50,7 @@
       user: "{{ magnum_trustee_domain_admin }}"
       role: "admin"
       auth: "{{ openstack_magnum_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       endpoint_type: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
       region_name: "{{ openstack_region_name }}"
diff --git a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
index 90b8efa43..d930f8dea 100644
--- a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
+++ b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml
@@ -27,13 +27,13 @@
       ansible.builtin.command: >
         {{ kolla_container_engine }} exec kolla_toolbox openstack
         --os-interface {{ openstack_interface }}
-        --os-auth-url {{ openstack_auth.auth_url }}
-        --os-project-domain-name {{ openstack_auth.domain_name }}
-        --os-project-name {{ openstack_auth.project_name }}
-        --os-username {{ openstack_auth.username }}
-        --os-password {{ openstack_auth.password }}
+        --os-auth-url {{ keystone_internal_url }}
+        --os-project-domain-name {{ default_project_domain_name }}
+        --os-project-name {{ keystone_admin_project }}
+        --os-username {{ keystone_admin_user }}
+        --os-password {{ keystone_admin_password }}
         --os-identity-api-version 3
-        --os-user-domain-name {{ openstack_auth.user_domain_name }}
+        --os-user-domain-name {{ default_user_domain_name }}
         --os-region-name {{ openstack_region_name }}
         {% if openstack_cacert != '' %}--os-cacert {{ openstack_cacert }}{% endif %}
         compute service list --format json --column Host --service nova-compute
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index 4f5c2f88d..2950dfcc4 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -94,7 +94,7 @@ proxyclient_address = {{ api_interface_address }}
 [ironic]
 username = {{ ironic_keystone_user }}
 password = {{ ironic_keystone_password }}
-auth_url = {{ openstack_auth.auth_url }}
+auth_url = {{ keystone_internal_url }}
 cafile = {{ openstack_cacert }}
 auth_type = password
 project_name = service
diff --git a/ansible/roles/octavia/defaults/main.yml b/ansible/roles/octavia/defaults/main.yml
index a75ee1770..4e53dac61 100644
--- a/ansible/roles/octavia/defaults/main.yml
+++ b/ansible/roles/octavia/defaults/main.yml
@@ -250,11 +250,8 @@ octavia_logging_debug: "{{ openstack_logging_debug }}"
 
 octavia_keystone_user: "octavia"
 
-# Project that Octavia will use to interact with other services.  Note that in
-# Train and earlier releases this was "admin".
-octavia_service_auth_project: "service"
-
 openstack_octavia_auth: "{{ openstack_auth }}"
+openstack_octavia_auth_cloud: "kolla-octavia-internal"
 
 octavia_api_workers: "{{ openstack_service_workers }}"
 octavia_healthmanager_health_workers: "{{ openstack_service_workers }}"
@@ -314,11 +311,7 @@ octavia_loadbalancer_topology: "SINGLE"
 
 # OpenStack auth used when registering resources for Octavia.
 octavia_user_auth:
-  auth_url: "{{ keystone_internal_url }}"
-  username: "octavia"
   password: "{{ octavia_keystone_password }}"
-  project_name: "{{ octavia_service_auth_project }}"
-  domain_name: "{{ default_project_domain_name }}"
 
 # Octavia amphora flavor.
 # See os_nova_flavor for details. Supported parameters:
diff --git a/ansible/roles/octavia/tasks/get_resources_info.yml b/ansible/roles/octavia/tasks/get_resources_info.yml
index ed4a4e4e1..a93aba43a 100644
--- a/ansible/roles/octavia/tasks/get_resources_info.yml
+++ b/ansible/roles/octavia/tasks/get_resources_info.yml
@@ -6,6 +6,7 @@
     module_name: openstack.cloud.compute_flavor_info
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -22,6 +23,7 @@
     module_name: openstack.cloud.project_info
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -40,6 +42,7 @@
     module_name: openstack.cloud.security_group
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -60,6 +63,7 @@
     module_name: openstack.cloud.networks_info
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
diff --git a/ansible/roles/octavia/tasks/hm-interface.yml b/ansible/roles/octavia/tasks/hm-interface.yml
index 60a7561fa..82ed4b0de 100644
--- a/ansible/roles/octavia/tasks/hm-interface.yml
+++ b/ansible/roles/octavia/tasks/hm-interface.yml
@@ -6,6 +6,7 @@
     module_name: openstack.cloud.port
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -26,13 +27,7 @@
     port_id: "{{ port_info.port.id }}"
   ansible.builtin.command: >
     {{ kolla_container_engine }} exec kolla_toolbox openstack
-    --os-interface {{ openstack_interface }}
-    --os-auth-url {{ octavia_user_auth.auth_url }}
-    --os-identity-api-version 3
-    --os-project-domain-name {{ octavia_user_auth.domain_name }}
-    --os-project-name {{ octavia_user_auth.project_name }}
-    --os-region-name {{ openstack_region_name }}
-    --os-username {{ octavia_user_auth.username }}
+    --os-cloud {{ openstack_octavia_auth_cloud }}
     --os-password {{ octavia_user_auth.password }}
     {% if openstack_cacert != '' %}--os-cacert {{ openstack_cacert }} {% endif %}
     port set --host {{ ansible_facts.nodename }} {{ port_id }}
diff --git a/ansible/roles/octavia/tasks/prepare.yml b/ansible/roles/octavia/tasks/prepare.yml
index 3e6ffa1a3..fb73c935e 100644
--- a/ansible/roles/octavia/tasks/prepare.yml
+++ b/ansible/roles/octavia/tasks/prepare.yml
@@ -6,6 +6,7 @@
     module_name: openstack.cloud.compute_flavor
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -30,6 +31,7 @@
     module_name: openstack.cloud.keypair
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -46,6 +48,7 @@
     module_name: openstack.cloud.project_info
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -61,6 +64,7 @@
     module_name: openstack.cloud.security_group
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -81,6 +85,7 @@
     module_name: openstack.cloud.security_group_rule
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -103,6 +108,7 @@
     module_name: openstack.cloud.network
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -125,6 +131,7 @@
     module_name: openstack.cloud.subnet
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
@@ -150,6 +157,7 @@
     module_name: openstack.cloud.router
     module_args:
       auth: "{{ octavia_user_auth }}"
+      cloud: "{{ openstack_octavia_auth_cloud }}"
       cacert: "{{ openstack_cacert }}"
       endpoint_type: "{{ openstack_interface }}"
       region_name: "{{ openstack_region_name }}"
diff --git a/ansible/roles/octavia/tasks/register.yml b/ansible/roles/octavia/tasks/register.yml
index a9f6a7775..6b763bc41 100644
--- a/ansible/roles/octavia/tasks/register.yml
+++ b/ansible/roles/octavia/tasks/register.yml
@@ -15,6 +15,7 @@
     module_args:
       name: "{{ item }}"
       auth: "{{ openstack_octavia_auth }}"
+      cloud: "{{ openstack_auth_cloud }}"
       endpoint_type: "{{ openstack_interface }}"
       cacert: "{{ openstack_cacert }}"
       region_name: "{{ openstack_region_name }}"
diff --git a/ansible/roles/service-ks-register/tasks/main.yml b/ansible/roles/service-ks-register/tasks/main.yml
index 242e597b6..2b875f808 100644
--- a/ansible/roles/service-ks-register/tasks/main.yml
+++ b/ansible/roles/service-ks-register/tasks/main.yml
@@ -13,6 +13,7 @@
           description: "{{ item.description | default(omit) }}"
           region_name: "{{ service_ks_register_region_name }}"
           auth: "{{ service_ks_register_auth }}"
+          cloud: "{{ openstack_auth_cloud }}"
           interface: "{{ service_ks_register_interface }}"
           cacert: "{{ service_ks_cacert }}"
           state: "{{ item.state | default('present') }}"
@@ -36,6 +37,7 @@
           region: "{{ service_ks_register_endpoint_region }}"
           region_name: "{{ service_ks_register_region_name }}"
           auth: "{{ service_ks_register_auth }}"
+          cloud: "{{ openstack_auth_cloud }}"
           interface: "{{ service_ks_register_interface }}"
           cacert: "{{ service_ks_cacert }}"
           state: "{{ item.state | default('present') }}"
@@ -59,6 +61,7 @@
           domain: "{{ service_ks_register_domain }}"
           region_name: "{{ service_ks_register_region_name }}"
           auth: "{{ service_ks_register_auth }}"
+          cloud: "{{ openstack_auth_cloud }}"
           interface: "{{ service_ks_register_interface }}"
           cacert: "{{ service_ks_cacert }}"
       with_items: "{{ service_ks_register_users | map(attribute='project') | unique | list }}"
@@ -83,6 +86,7 @@
           domain: "{{ service_ks_register_domain }}"
           region_name: "{{ service_ks_register_region_name }}"
           auth: "{{ service_ks_register_auth }}"
+          cloud: "{{ openstack_auth_cloud }}"
           interface: "{{ service_ks_register_interface }}"
           cacert: "{{ service_ks_cacert }}"
       with_items: "{{ service_ks_register_users }}"
@@ -101,6 +105,7 @@
           name: "{{ item }}"
           region_name: "{{ service_ks_register_region_name }}"
           auth: "{{ service_ks_register_auth }}"
+          cloud: "{{ openstack_auth_cloud }}"
           interface: "{{ service_ks_register_interface }}"
           cacert: "{{ service_ks_cacert }}"
       with_items: "{{ service_ks_register_users | map(attribute='role') | unique | list + service_ks_register_roles }}"
@@ -121,6 +126,7 @@
           system: "{{ item.system | default(omit) }}"
           region_name: "{{ service_ks_register_region_name }}"
           auth: "{{ service_ks_register_auth }}"
+          cloud: "{{ openstack_auth_cloud }}"
           interface: "{{ service_ks_register_interface }}"
           cacert: "{{ service_ks_cacert }}"
           state: "{{ item.state | default('present') }}"
diff --git a/doc/source/user/multi-regions.rst b/doc/source/user/multi-regions.rst
index dcee26d16..50040962c 100644
--- a/doc/source/user/multi-regions.rst
+++ b/doc/source/user/multi-regions.rst
@@ -71,14 +71,6 @@ the value of ``kolla_internal_fqdn`` in RegionOne:
 
    keystone_internal_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn_r1 }}:{{ keystone_public_port }}"
 
-   openstack_auth:
-       auth_url: "{{ keystone_internal_url }}"
-       username: "{{ keystone_admin_user }}"
-       password: "{{ keystone_admin_password }}"
-       user_domain_name: "{{ default_user_domain_name }}"
-       project_name: "{{ keystone_admin_project }}"
-       domain_name: "default"
-
 .. note::
 
    If the ``kolla_internal_vip_address`` and/or the
diff --git a/releasenotes/notes/toolbox-clouds-yaml-23faf5809aece04c.yaml b/releasenotes/notes/toolbox-clouds-yaml-23faf5809aece04c.yaml
new file mode 100644
index 000000000..7435b4aef
--- /dev/null
+++ b/releasenotes/notes/toolbox-clouds-yaml-23faf5809aece04c.yaml
@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    ``kolla-toolbox`` and all kolla_toolbox based invocations of Ansible
+    openstack.cloud modules have been reworked to use clouds.yaml.
+    ``openstack_auth`` variable is only used for passing keystone admin
+    password for services bootstrap runs.