Add support for encrypting backend Keystone HAProxy traffic

This patch introduces an optional backend encryption for Keystone
service. When used in conjunction with enabling TLS for service API
endpoints, network communcation will be encrypted end to end, from
client through HAProxy to the Keystone service.

Change-Id: I6351147ddaff8b2ae629179a9bc3bae2ebac9519
Partially-Implements: blueprint add-ssl-internal-network

25 files changed, 290 insertions, 64 deletions

diff --git a/ansible/certificates.yml b/ansible/certificates.yml
index f8021fcfc..4b6d2528d 100644
--- a/ansible/certificates.yml
+++ b/ansible/certificates.yml
@@ -1,4 +1,7 @@
 ---
+- import_playbook: gather-facts.yml
+  when: kolla_enable_tls_backend | default(false) | bool
+
 - name: Apply role certificates
   hosts: localhost
   roles:
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index d065f3bee..3d69bfb44 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -747,11 +747,18 @@ haproxy_user: "openstack"
 haproxy_enable_external_vip: "{{ 'no' if kolla_same_external_internal_vip | bool else 'yes' }}"
 kolla_enable_tls_internal: "no"
 kolla_enable_tls_external: "{{ kolla_enable_tls_internal if kolla_same_external_internal_vip | bool else 'no' }}"
-kolla_external_fqdn_cert: "{{ node_config }}/certificates/haproxy.pem"
-kolla_internal_fqdn_cert: "{{ node_config }}/certificates/haproxy-internal.pem"
-kolla_external_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy.crt"
-kolla_internal_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy-internal.crt"
+kolla_certificates_dir: "{{ node_config }}/certificates"
+kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy.pem"
+kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy-internal.pem"
+kolla_external_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy.crt"
+kolla_internal_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy-internal.crt"
 kolla_copy_ca_into_containers: "no"
+kolla_verify_tls_backend: "yes"
+haproxy_backend_cacert: "{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
+haproxy_backend_cacert_dir: "/etc/ssl/certs"
+kolla_enable_tls_backend: "no"
+kolla_tls_backend_cert: "{{ kolla_certificates_dir }}/backend-cert.pem"
+kolla_tls_backend_key: "{{ kolla_certificates_dir }}/backend-key.pem"
 
 ####################
 # Kibana options
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 483bc9449..65cc85e67 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -35,6 +35,9 @@ compute
 [baremetal:children]
 control
 
+[tls-backend:children]
+control
+
 [grafana:children]
 monitoring
 
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 98e4f1bb1..cbd415c3b 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -39,6 +39,9 @@ compute
 storage
 monitoring
 
+[tls-backend:children]
+control
+
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 [chrony-server:children]
diff --git a/ansible/roles/certificates/defaults/main.yml b/ansible/roles/certificates/defaults/main.yml
deleted file mode 100644
index a741e6a32..000000000
--- a/ansible/roles/certificates/defaults/main.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-# Directory on deploy node (localhost) in which certificates are generated.
-certificates_dir: "{{ node_config }}/certificates"
diff --git a/ansible/roles/certificates/tasks/generate.yml b/ansible/roles/certificates/tasks/generate.yml
index 6865c1239..0647bfe3e 100644
--- a/ansible/roles/certificates/tasks/generate.yml
+++ b/ansible/roles/certificates/tasks/generate.yml
@@ -1,21 +1,33 @@
 ---
 - name: Ensuring private internal directory exist
   file:
-    path: "{{ certificates_dir }}/private/internal"
+    path: "{{ kolla_certificates_dir }}/private/internal"
     state: "directory"
     recurse: yes
     mode: "0770"
 
 - name: Ensuring private external directory exist
   file:
-    path: "{{ certificates_dir }}/private/external"
+    path: "{{ kolla_certificates_dir }}/private/external"
     state: "directory"
     recurse: yes
     mode: "0770"
 
+- name: Ensuring backend certificate and key directories exist
+  file:
+    path: "{{ item | dirname }}"
+    state: "directory"
+    recurse: yes
+    mode: "0770"
+  when:
+    - kolla_enable_tls_backend | bool
+  with_items:
+    - "{{ kolla_tls_backend_cert }}"
+    - "{{ kolla_tls_backend_key }}"
+
 - name: Ensuring ca directory exist
   file:
-    path: "{{ certificates_dir }}/ca"
+    path: "{{ kolla_certificates_dir }}/ca"
     state: "directory"
     recurse: yes
     mode: "0770"
@@ -24,36 +36,36 @@
     - name: Creating external SSL configuration file
       template:
         src: "{{ item }}.j2"
-        dest: "{{ certificates_dir }}/{{ item }}"
+        dest: "{{ kolla_certificates_dir }}/{{ item }}"
         mode: "0660"
       with_items:
         - "openssl-kolla.cnf"
     - name: Creating external Key
       command: creates="{{ item }}" openssl genrsa -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/external/external.key"
+        - "{{ kolla_certificates_dir }}/private/external/external.key"
     - name: Setting permissions on external key
       file:
-        path: "{{ certificates_dir }}/private/external/external.key"
+        path: "{{ kolla_certificates_dir }}/private/external/external.key"
         mode: "0660"
         state: file
     - name: Creating external Server Certificate
       command: creates="{{ item }}" openssl req -new -nodes -sha256 -x509 \
-        -config {{ certificates_dir }}/openssl-kolla.cnf \
+        -config {{ kolla_certificates_dir }}/openssl-kolla.cnf \
         -days 3650 \
         -extensions v3_req \
-        -key {{ certificates_dir }}/private/external/external.key \
+        -key {{ kolla_certificates_dir }}/private/external/external.key \
         -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/external/external.crt"
+        - "{{ kolla_certificates_dir }}/private/external/external.crt"
     - name: Creating external CA Certificate File
       copy:
-        src: "{{ certificates_dir }}/private/external/external.crt"
+        src: "{{ kolla_certificates_dir }}/private/external/external.crt"
         dest: "{{ kolla_external_fqdn_cacert }}"
         mode: "0660"
     - name: Creating external Server PEM File
       assemble:
-        src: "{{ certificates_dir }}/private/external"
+        src: "{{ kolla_certificates_dir }}/private/external"
         dest: "{{ kolla_external_fqdn_cert }}"
         mode: "0660"
   when:
@@ -62,14 +74,14 @@
 - block:
     - name: Copy the external certificate crt to be the internal when internal + external are same network
       copy:
-        src: "{{ certificates_dir }}/private/external/external.crt"
-        dest: "{{ certificates_dir }}/private/internal/internal.crt"
+        src: "{{ kolla_certificates_dir }}/private/external/external.crt"
+        dest: "{{ kolla_certificates_dir }}/private/internal/internal.crt"
         remote_src: yes
         mode: "0660"
     - name: Copy the external certificate key to be the internal when internal + external are same network
       copy:
-        src: "{{ certificates_dir }}/private/external/external.key"
-        dest: "{{ certificates_dir }}/private/internal/internal.key"
+        src: "{{ kolla_certificates_dir }}/private/external/external.key"
+        dest: "{{ kolla_certificates_dir }}/private/internal/internal.key"
         remote_src: yes
         mode: "0660"
     - name: Copy the external PEM file to be the internal when internal + external are same network
@@ -93,38 +105,72 @@
     - name: Creating internal SSL configuration file
       template:
         src: "{{ item }}.j2"
-        dest: "{{ certificates_dir }}/{{ item }}"
+        dest: "{{ kolla_certificates_dir }}/{{ item }}"
         mode: "0660"
       with_items:
         - "openssl-kolla-internal.cnf"
     - name: Creating internal Key
       command: creates="{{ item }}" openssl genrsa -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/internal/internal.key"
+        - "{{ kolla_certificates_dir }}/private/internal/internal.key"
     - name: Setting permissions on internal key
       file:
-        path: "{{ certificates_dir }}/private/internal/internal.key"
+        path: "{{ kolla_certificates_dir }}/private/internal/internal.key"
         mode: "0660"
         state: file
     - name: Creating internal Server Certificate
       command: creates="{{ item }}" openssl req -new -nodes -sha256 -x509 \
-        -config {{ certificates_dir }}/openssl-kolla-internal.cnf \
+        -config {{ kolla_certificates_dir }}/openssl-kolla-internal.cnf \
         -days 3650 \
         -extensions v3_req \
-        -key {{ certificates_dir }}/private/internal/internal.key \
+        -key {{ kolla_certificates_dir }}/private/internal/internal.key \
         -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/internal/internal.crt"
+        - "{{ kolla_certificates_dir }}/private/internal/internal.crt"
     - name: Creating internal CA Certificate File
       copy:
-        src: "{{ certificates_dir }}/private/internal/internal.crt"
+        src: "{{ kolla_certificates_dir }}/private/internal/internal.crt"
         dest: "{{ kolla_internal_fqdn_cacert }}"
         mode: "0660"
     - name: Creating internal Server PEM File
       assemble:
-        src: "{{ certificates_dir }}/private/internal"
+        src: "{{ kolla_certificates_dir }}/private/internal"
         dest: "{{ kolla_internal_fqdn_cert }}"
         mode: "0660"
   when:
     - kolla_enable_tls_internal | bool
     - not kolla_same_external_internal_vip | bool
+
+- block:
+    - name: Creating backend SSL configuration file
+      template:
+        src: "{{ item }}.j2"
+        dest: "{{ kolla_certificates_dir }}/{{ item }}"
+        mode: "0660"
+      with_items:
+        - "openssl-kolla-backend.cnf"
+    - name: Creating backend Key
+      command: creates="{{ item }}" openssl genrsa -out {{ item }}
+      with_items:
+        - "{{ kolla_tls_backend_key }}"
+    - name: Setting permissions on backend key
+      file:
+        path: "{{ kolla_tls_backend_key }}"
+        mode: "0660"
+        state: file
+    - name: Creating backend Server Certificate
+      command: creates="{{ item }}" openssl req -new -nodes -sha256 -x509 \
+        -config {{ kolla_certificates_dir }}/openssl-kolla-backend.cnf \
+        -days 3650 \
+        -extensions v3_req \
+        -key {{ kolla_tls_backend_key }} \
+        -out {{ item }}
+      with_items:
+        - "{{ kolla_tls_backend_cert }}"
+    - name: Creating backend Certificate file to be included in container trusted ca-certificates
+      copy:
+        src: "{{ kolla_tls_backend_cert }}"
+        dest: "{{ kolla_certificates_dir }}/ca/backend-cert.crt"
+        mode: "0660"
+  when:
+    - kolla_enable_tls_backend | bool
diff --git a/ansible/roles/certificates/templates/openssl-kolla-backend.cnf.j2 b/ansible/roles/certificates/templates/openssl-kolla-backend.cnf.j2
new file mode 100644
index 000000000..bd862832a
--- /dev/null
+++ b/ansible/roles/certificates/templates/openssl-kolla-backend.cnf.j2
@@ -0,0 +1,18 @@
+[req]
+prompt = no
+distinguished_name = req_distinguished_name
+req_extensions = v3_req
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = NC
+localityName = RTP
+organizationalUnitName = kolla
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+{% for host in groups['tls-backend']%}
+IP.{{ loop.index }} = {{ 'api' | kolla_address(host) }}
+{% endfor %}
diff --git a/ansible/roles/haproxy-config/defaults/main.yml b/ansible/roles/haproxy-config/defaults/main.yml
index 7c2e54895..97ef2a099 100644
--- a/ansible/roles/haproxy-config/defaults/main.yml
+++ b/ansible/roles/haproxy-config/defaults/main.yml
@@ -13,3 +13,4 @@ haproxy_backend_http_extra: []
 haproxy_backend_tcp_extra: []
 
 haproxy_health_check: "check inter 2000 rise 2 fall 5"
+haproxy_health_check_ssl: "check check-ssl inter 2000 rise 2 fall 5"
diff --git a/ansible/roles/haproxy-config/templates/haproxy_single_service_listen.cfg.j2 b/ansible/roles/haproxy-config/templates/haproxy_single_service_listen.cfg.j2
index b1b133dad..16076da50 100644
--- a/ansible/roles/haproxy-config/templates/haproxy_single_service_listen.cfg.j2
+++ b/ansible/roles/haproxy-config/templates/haproxy_single_service_listen.cfg.j2
@@ -10,7 +10,7 @@ userlist {{ service_name }}-user
 {%- macro listen_macro(service_name, service_port, listen_port,
                        service_mode, external,
                        haproxy_http_extra, haproxy_tcp_extra, host_group,
-                       custom_member_list, auth_user, auth_pass) %}
+                       custom_member_list, auth_user, auth_pass, tls_backend) %}
 listen {{ service_name }}
     {% if service_mode == 'redirect' %}
     mode http
@@ -59,10 +59,21 @@ listen {{ service_name }}
     {{ custom_member }}
             {% endfor %}
         {% else %}
+            {% set backend_tls_info = '' %}
+            {% if tls_backend|bool %}
+                {% set haproxy_health_check_final = haproxy_health_check_ssl %}
+                {% if kolla_verify_tls_backend|bool %}
+                    {% set backend_tls_info = 'ssl verify required ca-file %s'|format(haproxy_backend_cacert) %}
+                {% else %}
+                    {% set backend_tls_info = 'ssl verify none' %}
+                {% endif %}
+            {% else %}
+                {% set haproxy_health_check_final = haproxy_health_check %}
+            {% endif %}
             {% for host in groups[host_group] %}
                 {% set host_name = hostvars[host]['ansible_hostname'] %}
                 {% set host_ip = 'api' | kolla_address(host) %}
-    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check }}
+    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check_final }} {{ backend_tls_info }}
             {% endfor %}
         {% endif %}
     {% endif %}
@@ -86,6 +97,7 @@ listen {{ service_name }}
         {# Additional options can be defined in config, and are additive to the global extras #}
         {% set haproxy_tcp_extra = haproxy_service.frontend_tcp_extra|default([]) + haproxy_service.backend_tcp_extra|default([]) + haproxy_frontend_tcp_extra + haproxy_backend_tcp_extra %}
         {% set haproxy_http_extra = haproxy_service.frontend_http_extra|default([]) + haproxy_service.backend_http_extra|default([]) + haproxy_frontend_http_extra + haproxy_backend_http_extra %}
+        {% set tls_backend = haproxy_service.tls_backend|default(false) %}
         {# Allow for basic auth #}
         {% set auth_user = haproxy_service.auth_user|default() %}
         {% set auth_pass = haproxy_service.auth_pass|default() %}
@@ -94,6 +106,6 @@ listen {{ service_name }}
         {% endif %}
 {{ listen_macro(haproxy_name, haproxy_service.port, listen_port,
                 mode, external, haproxy_http_extra, haproxy_tcp_extra,
-                host_group, custom_member_list, auth_user, auth_pass) }}
+                host_group, custom_member_list, auth_user, auth_pass, tls_backend) }}
     {% endif %}
 {%- endfor -%}
diff --git a/ansible/roles/haproxy-config/templates/haproxy_single_service_split.cfg.j2 b/ansible/roles/haproxy-config/templates/haproxy_single_service_split.cfg.j2
index 27b63e4da..ec269a307 100644
--- a/ansible/roles/haproxy-config/templates/haproxy_single_service_split.cfg.j2
+++ b/ansible/roles/haproxy-config/templates/haproxy_single_service_split.cfg.j2
@@ -53,7 +53,7 @@ frontend {{ service_name }}_front
 
 {%- macro backend_macro(service_name, listen_port, service_mode, host_group,
                         custom_member_list, backend_http_extra,
-                        backend_tcp_extra, auth_user, auth_pass) %}
+                        backend_tcp_extra, auth_user, auth_pass, tls_backend) %}
 backend {{ service_name }}_back
     {% if service_mode == 'redirect' %}
     mode http
@@ -79,10 +79,21 @@ backend {{ service_name }}_back
     {{ custom_member }}
         {% endfor %}
     {% else %}
+        {% set backend_tls_info = '' %}
+        {% if tls_backend|bool %}
+            {% set haproxy_health_check_final = haproxy_health_check_ssl %}
+            {% if kolla_verify_tls_backend|bool %}
+                {% set backend_tls_info = 'ssl verify required ca-file %s'|format(haproxy_backend_cacert) %}
+            {% else %}
+                {% set backend_tls_info = 'ssl verify none' %}
+            {% endif %}
+        {% else %}
+            {% set haproxy_health_check_final = haproxy_health_check %}
+        {% endif %}
         {% for host in groups[host_group] %}
             {% set host_name = hostvars[host]['ansible_hostname'] %}
             {% set host_ip = 'api' | kolla_address(host) %}
-    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check }}
+    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check_final }} {{ backend_tls_info }}
         {% endfor %}
     {% endif %}
 {% endmacro %}
@@ -107,6 +118,7 @@ backend {{ service_name }}_back
         {% set backend_tcp_extra = haproxy_service.backend_tcp_extra|default([]) %}
         {% set frontend_http_extra = haproxy_service.frontend_http_extra|default([]) + haproxy_frontend_http_extra %}
         {% set backend_http_extra = haproxy_service.backend_http_extra|default([]) %}
+        {% set tls_backend = haproxy_service.tls_backend|default(false) %}
         {# Allow for basic auth #}
         {% set auth_user = haproxy_service.auth_user|default() %}
         {% set auth_pass = haproxy_service.auth_pass|default() %}
@@ -119,7 +131,7 @@ backend {{ service_name }}_back
         {% if haproxy_service.mode != 'redirect' %}
 {{ backend_macro(haproxy_name, listen_port, mode, host_group,
                  custom_member_list, backend_http_extra, backend_tcp_extra,
-                 auth_user, auth_pass) }}
+                 auth_user, auth_pass, tls_backend) }}
         {% endif %}
     {% endif %}
 {%- endfor -%}
diff --git a/ansible/roles/haproxy/tasks/config.yml b/ansible/roles/haproxy/tasks/config.yml
index fd215a32d..33c46c143 100644
--- a/ansible/roles/haproxy/tasks/config.yml
+++ b/ansible/roles/haproxy/tasks/config.yml
@@ -142,6 +142,20 @@
   notify:
     - Restart haproxy container
 
+- name: Copying over extra CA certificates
+  vars:
+    service: "{{ haproxy_services['haproxy'] }}"
+  become: true
+  copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/haproxy/ca-certificates"
+    mode: "0644"
+  when:
+    - inventory_hostname in groups[service.group]
+    - kolla_copy_ca_into_containers | bool
+  notify:
+    - Restart haproxy container
+
 - name: Copying over haproxy start script
   vars:
     service: "{{ haproxy_services['haproxy'] }}"
diff --git a/ansible/roles/haproxy/templates/haproxy_main.cfg.j2 b/ansible/roles/haproxy/templates/haproxy_main.cfg.j2
index 4fa141783..815fc0536 100644
--- a/ansible/roles/haproxy/templates/haproxy_main.cfg.j2
+++ b/ansible/roles/haproxy/templates/haproxy_main.cfg.j2
@@ -18,6 +18,9 @@ global
     ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
     tune.ssl.default-dh-param 4096
     {% endif %}
+    {% if kolla_enable_tls_internal | bool or kolla_enable_tls_external | bool %}
+    ca-base {{ haproxy_backend_cacert_dir }}
+    {% endif %}
 
 defaults
     log global
diff --git a/ansible/roles/keystone/defaults/main.yml b/ansible/roles/keystone/defaults/main.yml
index 717ee20ff..f58a7fc49 100644
--- a/ansible/roles/keystone/defaults/main.yml
+++ b/ansible/roles/keystone/defaults/main.yml
@@ -14,18 +14,21 @@ keystone_services:
         enabled: "{{ enable_keystone }}"
         mode: "http"
         external: false
+        tls_backend: "{{ keystone_enable_tls_backend }}"
         port: "{{ keystone_public_port }}"
         listen_port: "{{ keystone_public_listen_port }}"
       keystone_external:
         enabled: "{{ enable_keystone }}"
         mode: "http"
         external: true
+        tls_backend: "{{ keystone_enable_tls_backend }}"
         port: "{{ keystone_public_port }}"
         listen_port: "{{ keystone_public_listen_port }}"
       keystone_admin:
         enabled: "{{ enable_keystone }}"
         mode: "http"
         external: false
+        tls_backend: "{{ keystone_enable_tls_backend }}"
         port: "{{ keystone_admin_port }}"
         listen_port: "{{ keystone_admin_listen_port }}"
   keystone-ssh:
@@ -141,3 +144,8 @@ keystone_ks_services:
       - {'interface': 'admin', 'url': '{{ keystone_admin_url }}'}
       - {'interface': 'internal', 'url': '{{ keystone_internal_url }}'}
       - {'interface': 'public', 'url': '{{ keystone_public_url }}'}
+
+####################
+# TLS
+####################
+keystone_enable_tls_backend: "{{ kolla_enable_tls_backend }}"
diff --git a/ansible/roles/keystone/tasks/config.yml b/ansible/roles/keystone/tasks/config.yml
index d31d1d201..b47b77f4f 100644
--- a/ansible/roles/keystone/tasks/config.yml
+++ b/ansible/roles/keystone/tasks/config.yml
@@ -38,19 +38,9 @@
   run_once: True
   register: keystone_domain_directory
 
-- name: Copying over extra CA certificates
-  become: true
-  copy:
-    src: "{{ node_config }}/certificates/ca/"
-    dest: "{{ node_config_directory }}/{{ item.key }}/ca-certificates"
-    mode: "0644"
+- include_tasks: copy-certs.yml
   when:
-    - item.value.enabled | bool
-    - inventory_hostname in groups[item.value.group]
-    - kolla_copy_ca_into_containers | bool
-  with_dict: "{{ keystone_services }}"
-  notify:
-    - "Restart {{ item.key }} container"
+    - kolla_copy_ca_into_containers | bool or keystone_enable_tls_backend | bool
 
 - name: Copying over config.json files for services
   template:
diff --git a/ansible/roles/keystone/tasks/copy-certs.yml b/ansible/roles/keystone/tasks/copy-certs.yml
new file mode 100644
index 000000000..7fd52ba30
--- /dev/null
+++ b/ansible/roles/keystone/tasks/copy-certs.yml
@@ -0,0 +1,6 @@
+---
+- name: "Copy certificates and keys for {{ project_name }}"
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ keystone_services }}"
diff --git a/ansible/roles/keystone/templates/keystone.json.j2 b/ansible/roles/keystone/templates/keystone.json.j2
index 4269d7e0f..6637c5f55 100644
--- a/ansible/roles/keystone/templates/keystone.json.j2
+++ b/ansible/roles/keystone/templates/keystone.json.j2
@@ -34,7 +34,19 @@
             "dest": "/etc/{{ keystone_dir }}/wsgi-keystone.conf",
             "owner": "keystone",
             "perm": "0600"
-        }
+        }{% if keystone_enable_tls_backend | bool %},
+        {
+            "source": "{{ container_config_directory }}/keystone-cert.pem",
+            "dest": "/etc/keystone/certs/keystone-cert.pem",
+            "owner": "keystone",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/keystone-key.pem",
+            "dest": "/etc/keystone/certs/keystone-key.pem",
+            "owner": "keystone",
+            "perm": "0600"
+        }{% endif %}
     ],
     "permissions": [
         {
diff --git a/ansible/roles/keystone/templates/wsgi-keystone.conf.j2 b/ansible/roles/keystone/templates/wsgi-keystone.conf.j2
index 97c12ada5..17399814e 100644
--- a/ansible/roles/keystone/templates/wsgi-keystone.conf.j2
+++ b/ansible/roles/keystone/templates/wsgi-keystone.conf.j2
@@ -5,6 +5,9 @@
 {% set python_path = '/var/lib/kolla/venv/lib/python' + distro_python_version + '/site-packages' %}
 {% endif %}
 {% set binary_path = '/usr/bin' if keystone_install_type == 'binary' else '/var/lib/kolla/venv/bin' %}
+{% if keystone_enable_tls_backend | bool %}
+LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so
+{% endif %}
 Listen {{ api_interface_address | put_address_in_context('url') }}:{{ keystone_public_listen_port }}
 Listen {{ api_interface_address | put_address_in_context('url') }}:{{ keystone_admin_listen_port }}
 
@@ -42,6 +45,12 @@ LogLevel info
     ErrorLog "{{ keystone_log_dir }}/keystone-apache-public-error.log"
     LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
     CustomLog "{{ keystone_log_dir }}/keystone-apache-public-access.log" logformat
+
+{% if keystone_enable_tls_backend | bool %}
+    SSLEngine on
+    SSLCertificateFile /etc/keystone/certs/keystone-cert.pem
+    SSLCertificateKeyFile /etc/keystone/certs/keystone-key.pem
+{% endif %}
 </VirtualHost>
 
 <VirtualHost *:{{ keystone_admin_listen_port }}>
@@ -56,4 +65,10 @@ LogLevel info
     ErrorLog "{{ keystone_log_dir }}/keystone-apache-admin-error.log"
     LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
     CustomLog "{{ keystone_log_dir }}/keystone-apache-admin-access.log" logformat
+
+{% if keystone_enable_tls_backend | bool %}
+    SSLEngine on
+    SSLCertificateFile /etc/keystone/certs/keystone-cert.pem
+    SSLCertificateKeyFile /etc/keystone/certs/keystone-key.pem
+{% endif %}
 </VirtualHost>
diff --git a/ansible/roles/service-cert-copy/tasks/main.yml b/ansible/roles/service-cert-copy/tasks/main.yml
new file mode 100644
index 000000000..c8af5d50a
--- /dev/null
+++ b/ansible/roles/service-cert-copy/tasks/main.yml
@@ -0,0 +1,54 @@
+---
+- name: "{{ project_name }} | Copying over extra CA certificates"
+  become: true
+  copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ item.key }}/ca-certificates"
+    mode: "0644"
+  when:
+    - kolla_copy_ca_into_containers | bool
+  with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: "{{ project_name }} | Copying over backend internal TLS certificate"
+  vars:
+    certs:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/{{ project_name }}-cert.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
+      - "{{ kolla_certificates_dir }}/{{ project_name }}-cert.pem"
+      - "{{ kolla_tls_backend_cert }}"
+    backend_tls_cert: "{{ lookup('first_found', certs) }}"
+  copy:
+    src: "{{ backend_tls_cert }}"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ project_name }}-cert.pem"
+    mode: "0644"
+  become: true
+  when:
+    - item.value.haproxy is defined
+    - item.value.haproxy.values() | selectattr('enabled', 'defined') | map(attribute='enabled') | map('bool') | select | list | length > 0
+    - item.value.haproxy.values() | selectattr('tls_backend', 'defined') | map(attribute='tls_backend') | map('bool') | select | list | length > 0
+  with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: "{{ project_name }} | Copying over backend internal TLS key"
+  vars:
+    keys:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/{{ project_name }}-key.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
+      - "{{ kolla_certificates_dir }}/{{ project_name }}-key.pem"
+      - "{{ kolla_tls_backend_key }}"
+    backend_tls_key: "{{ lookup('first_found', keys) }}"
+  copy:
+    src: "{{ backend_tls_key }}"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ project_name }}-key.pem"
+    mode: "0600"
+  become: true
+  when:
+    - item.value.haproxy is defined
+    - item.value.haproxy.values() | selectattr('enabled', 'defined') | map(attribute='enabled') | map('bool') | select | list | length > 0
+    - item.value.haproxy.values() | selectattr('tls_backend', 'defined') | map(attribute='tls_backend') | map('bool') | select | list | length > 0
+  with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - "Restart {{ item.key }} container"
diff --git a/doc/source/admin/advanced-configuration.rst b/doc/source/admin/advanced-configuration.rst
index 765ed2f69..34c21a06e 100644
--- a/doc/source/admin/advanced-configuration.rst
+++ b/doc/source/admin/advanced-configuration.rst
@@ -99,12 +99,12 @@ The default for TLS is disabled, to enable TLS networking:
 .. code-block:: yaml
 
    kolla_enable_tls_external: "yes"
-   kolla_external_fqdn_cert: "{{ node_config }}/certificates/mycert.pem"
+   kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/mycert.pem"
 
    and/or
 
    kolla_enable_tls_internal: "yes"
-   kolla_internal_fqdn_cert: "{{ node_config }}/certificates/mycert-internal.pem"
+   kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/mycert-internal.pem"
 
 
 .. note::
@@ -181,7 +181,7 @@ service containers to enable trust for those CA certificates. This is required
 for any certificates that are either self-signed or signed by a private CA,
 and are not already present in the service image trust store.
 
-All certificate file names will have the "kolla-customca-" prefix appended to
+All certificate file names will have the "kolla-customca-" prefix prepended to
 it when it is copied into the containers. For example, if a certificate file is
 named "internal.crt", it will be named "kolla-customca-internal.crt" in the
 containers.
@@ -192,6 +192,11 @@ the ``/usr/local/share/ca-certificates/`` directory.
 For Centos and Red Hat Linux containers, the certificate files will be copied
 to the ``/etc/pki/ca-trust/source/anchors/`` directory.
 
+In addition, the ``openstack_cacert`` should be configured with the path to
+the cacert in the container. For example, if the self-signed certificate task
+was used and the deployment is on ubuntu, the path would be:
+"/etc/pki/ca-trust/source/anchors/kolla-customca-haproxy-internal.crt"
+
 .. _service-config:
 
 OpenStack Service Configuration in Kolla
diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml
index 524edaa24..f40fe35c0 100644
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@@ -182,11 +182,18 @@
 # allow clients to perform authentication.
 #kolla_enable_tls_internal: "no"
 #kolla_enable_tls_external: "{{ kolla_enable_tls_internal if kolla_same_external_internal_vip | bool else 'no' }}"
-#kolla_external_fqdn_cert: "{{ node_config }}/certificates/haproxy.pem"
-#kolla_internal_fqdn_cert: "{{ node_config }}/certificates/haproxy-internal.pem"
-#kolla_external_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy.crt"
-#kolla_internal_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy-internal.crt"
+#kolla_certificates_dir: "{{ node_config }}/certificates"
+#kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy.pem"
+#kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy-internal.pem"
+#kolla_external_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy.crt"
+#kolla_internal_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy-internal.crt"
 #kolla_copy_ca_into_containers: "no"
+#kolla_verify_tls_backend: "yes"
+#haproxy_backend_cacert: "{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
+#haproxy_backend_cacert_dir: "/etc/ssl/certs"
+#kolla_enable_tls_backend: "no"
+#kolla_tls_backend_cert: "{{ kolla_certificates_dir }}/backend-cert.pem"
+#kolla_tls_backend_key: "{{ kolla_certificates_dir }}/backend-key.pem"
 
 ################
 # Region options
diff --git a/releasenotes/notes/copy-certificate-authority-into-containers-860cbda3384dd731.yaml b/releasenotes/notes/copy-certificate-authority-into-containers-860cbda3384dd731.yaml
index 78c7e11db..f5019df18 100644
--- a/releasenotes/notes/copy-certificate-authority-into-containers-860cbda3384dd731.yaml
+++ b/releasenotes/notes/copy-certificate-authority-into-containers-860cbda3384dd731.yaml
@@ -12,10 +12,8 @@ features:
 
 issues:
   - |
-    Python <= 2.7.9 will not trust self-signed or privately signed CAs even
-    if they are added into the OS trusted CA folder and update-ca-trust is
-    executed. This is also true for the Python Requests library, regardless of
-    Python version. For services that run Python <= 2.7.9 or rely on the
-    Python Requests library, either CA verification must be explicitly disabled
-    in the service or the path to the CA certificate must be configured using
-    the ``openstack_cacert`` parameter.
+    Python Requests library will not trust self-signed or privately signed CAs
+    even if they are added into the OS trusted CA folder and update-ca-trust is
+    executed. For services that rely on the Python Requests library, either CA
+    verification must be explicitly disabled in the service or the path to the
+    CA certificate must be configured using the ``openstack_cacert`` parameter.
diff --git a/releasenotes/notes/encrypt-backend-haproxy-keystone-fb96285d74fb464c.yaml b/releasenotes/notes/encrypt-backend-haproxy-keystone-fb96285d74fb464c.yaml
new file mode 100644
index 000000000..1b7807270
--- /dev/null
+++ b/releasenotes/notes/encrypt-backend-haproxy-keystone-fb96285d74fb464c.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Added configuration options to enable backend TLS encryption from HAProxy
+    to the Keystone service. When used in conjunction with enabling TLS for
+    service API endpoints, network communcation will be encrypted end to end,
+    from client through HAProxy to the Keystone service.
diff --git a/tests/check-config.sh b/tests/check-config.sh
index 4bc081660..de6577e08 100755
--- a/tests/check-config.sh
+++ b/tests/check-config.sh
@@ -16,6 +16,8 @@ function check_config {
     for f in $(sudo find /etc/kolla \
                 -not -regex /etc/kolla/config.* \
                 -not -regex /etc/kolla/certificates.* \
+                -not -regex .*pem \
+                -not -regex .*key \
                 -not -regex ".*ca-certificates.*" \
                 -not -path /etc/kolla \
                 -not -name admin-openrc.sh \
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index e1217ba4b..3b0ac0239 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -117,8 +117,8 @@ ceph_nova_user: "cinder"
 {% if tls_enabled %}
 kolla_enable_tls_external: "yes"
 kolla_enable_tls_internal: "yes"
-kolla_verify_internal_ca_certs: "no"
 kolla_copy_ca_into_containers: "yes"
+kolla_enable_tls_backend: "yes"
 {% if base_distro == "ubuntu" or base_distro == "debian" %}
 openstack_cacert: "/usr/local/share/ca-certificates/kolla-customca-haproxy-internal.crt"
 {% endif %}
diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2
index 6b0c01e8c..fbc15f1a1 100644
--- a/tests/templates/inventory.j2
+++ b/tests/templates/inventory.j2
@@ -53,6 +53,9 @@ compute
 storage
 monitoring
 
+[tls-backend:children]
+control
+
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 [chrony-server:children]