Add full support for fernet

This addresses the ansible aspects of fernet key bootstrapping as
well as distributed key rotation.

- Bootstrapping is handled in the same way as keystone bootstrap.
- A new keystone-fernet and keystone-ssh container is created to allow
  the nodes to communicate with each other (taken from nova-ssh).
- The keystone-fernet is a keystone container with crontab installed.
  This will handle key rotations through keystone-manage and trigger
  an rsync to push new tokens to other nodes.
- Key rotation is setup to be balanced across the keystone nodes using
  a round-robbin style. This ensures that any node failures will not
  stop the keys from rotating. This is configured by a desired token
  expiration time which then determines the cron scheduling for each
  node as well as the number of fernet tokens in rotation.
- Ability for recovered node to resync with the cluster. When a node
  starts it will run sanity checks to ensure that its fernet tokens
  are not stale. If they are it will rsync with other nodes to ensure
  its tokens are up to date.

The Docker component is implemented in:
  https://review.openstack.org/#/c/349366

Change-Id: I15052c25a1d1149d364236f10ced2e2346119738
Implements: blueprint keystone-fernet-token

24 files changed, 384 insertions, 12 deletions

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 97943fd14..d6e25ad77 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -131,6 +131,7 @@ haproxy_stats_port: "1984"
 
 keystone_public_port: "5000"
 keystone_admin_port: "35357"
+keystone_ssh_port: "8023"
 
 glance_api_port: "9292"
 glance_registry_port: "9191"
@@ -282,6 +283,13 @@ keystone_internal_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ k
 keystone_public_url: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ keystone_public_port }}/v3"
 
 
+#####################################
+# Keystone - Identity Service options
+#####################################
+keystone_token_provider: "uuid"
+fernet_token_expiry: 86400
+
+
 #######################
 # Glance options
 #######################
diff --git a/ansible/roles/keystone/defaults/main.yml b/ansible/roles/keystone/defaults/main.yml
index ba8ec9c37..0fda77a9a 100644
--- a/ansible/roles/keystone/defaults/main.yml
+++ b/ansible/roles/keystone/defaults/main.yml
@@ -10,12 +10,27 @@ keystone_database_address: "{{ kolla_internal_fqdn }}:{{ database_port }}"
 
 
 ####################
+# Fernet
+####################
+keystone_username: "keystone"
+keystone_groupname: "keystone"
+
+
+####################
 # Docker
 ####################
 keystone_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-keystone"
 keystone_tag: "{{ openstack_release }}"
 keystone_image_full: "{{ keystone_image }}:{{ keystone_tag }}"
 
+keystone_fernet_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-keystone-fernet"
+keystone_fernet_tag: "{{ openstack_release }}"
+keystone_fernet_image_full: "{{ keystone_fernet_image }}:{{ keystone_fernet_tag }}"
+
+keystone_ssh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-keystone-ssh"
+keystone_ssh_tag: "{{ openstack_release }}"
+keystone_ssh_image_full: "{{ keystone_ssh_image }}:{{ keystone_ssh_tag }}"
+
 
 ####################
 # OpenStack
diff --git a/ansible/roles/keystone/files/fernet_rotate_cron_generator.py b/ansible/roles/keystone/files/fernet_rotate_cron_generator.py
new file mode 100644
index 000000000..da468a851
--- /dev/null
+++ b/ansible/roles/keystone/files/fernet_rotate_cron_generator.py
@@ -0,0 +1,107 @@
+#!/usr/bin/python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module creates a list of cron intervals for a node in a group of nodes
+# to ensure each node runs a cron in round robbin style.
+
+from __future__ import print_function
+import argparse
+import json
+import sys
+
+MINUTE_SPAN = 1
+HOUR_SPAN = 60
+DAY_SPAN = 24 * HOUR_SPAN
+WEEK_SPAN = 7 * DAY_SPAN
+
+
+def json_exit(msg=None, failed=False, changed=False):
+    if type(msg) is not dict:
+        msg = {'msg': str(msg)}
+    msg.update({'failed': failed, 'changed': changed})
+    print(json.dumps(msg))
+    sys.exit()
+
+
+def generate(host_index, total_hosts, total_rotation_mins):
+    min = '*'
+    hour = '*'
+    day = '*'
+    crons = []
+
+    if host_index >= total_hosts:
+        return crons
+
+    rotation_frequency = total_rotation_mins // total_hosts
+    cron_min = rotation_frequency * host_index
+
+    # Build crons for a week period
+    if total_rotation_mins == WEEK_SPAN:
+        day = cron_min // DAY_SPAN
+        hour = (cron_min % DAY_SPAN) // HOUR_SPAN
+        min = cron_min % HOUR_SPAN
+        crons.append({'min': min, 'hour': hour, 'day': day})
+
+    # Build crons for a day period
+    elif total_rotation_mins == DAY_SPAN:
+        hour = cron_min // HOUR_SPAN
+        min = cron_min % HOUR_SPAN
+        crons.append({'min': min, 'hour': hour, 'day': day})
+
+    # Build crons for multiple of an hour
+    elif total_rotation_mins % HOUR_SPAN == 0:
+        for multiple in range(1, DAY_SPAN // total_rotation_mins + 1):
+            time = cron_min
+            if multiple > 1:
+                time += total_rotation_mins * (multiple - 1)
+
+            hour = time // HOUR_SPAN
+            min = time % HOUR_SPAN
+            crons.append({'min': min, 'hour': hour, 'day': day})
+
+    # Build crons for multiple of a minute
+    elif total_rotation_mins % MINUTE_SPAN == 0:
+        for multiple in range(1, HOUR_SPAN // total_rotation_mins + 1):
+            time = cron_min
+            if multiple > 1:
+                time += total_rotation_mins * (multiple - 1)
+
+            min = time // MINUTE_SPAN
+            crons.append({'min': min, 'hour': hour, 'day': day})
+
+    return crons
+
+
+def main():
+    parser = argparse.ArgumentParser(description='''Creates a list of cron
+        intervals for a node in a group of nodes to ensure each node runs
+        a cron in round robbin style.''')
+    parser.add_argument('-t', '--time',
+                        help='Time in seconds for a token rotation cycle',
+                        required=True,
+                        type=int)
+    parser.add_argument('-i', '--index',
+                        help='Index of host starting from 0',
+                        required=True,
+                        type=int)
+    parser.add_argument('-n', '--number',
+                        help='Number of tokens that should exist',
+                        required=True,
+                        type=int)
+    args = parser.parse_args()
+    json_exit({'cron_jobs': generate(args.index, args.number, args.time)})
+
+
+if __name__ == "__main__":
+    main()
diff --git a/ansible/roles/keystone/tasks/config.yml b/ansible/roles/keystone/tasks/config.yml
index cfca7dbad..65ceb1a95 100644
--- a/ansible/roles/keystone/tasks/config.yml
+++ b/ansible/roles/keystone/tasks/config.yml
@@ -14,6 +14,8 @@
     recurse: yes
   with_items:
     - "keystone"
+    - "keystone-fernet"
+    - "keystone-ssh"
 
 - name: Creating Keystone Domain directory
   file:
@@ -30,6 +32,8 @@
     dest: "{{ node_config_directory }}/{{ item }}/config.json"
   with_items:
     - "keystone"
+    - "keystone-fernet"
+    - "keystone-ssh"
 
 - name: Copying over keystone.conf
   merge_configs:
@@ -45,6 +49,8 @@
     dest: "{{ node_config_directory }}/{{ item }}/keystone.conf"
   with_items:
     - "keystone"
+    - "keystone-fernet"
+    - "keystone-ssh"
 
 - name: Copying Keystone Domain specific settings
   copy:
@@ -68,3 +74,34 @@
     - "{{ node_custom_config }}/keystone/{{ inventory_hostname }}/wsgi-keystone.conf"
     - "{{ node_custom_config }}/keystone/wsgi-keystone.conf"
     - "wsgi-keystone.conf.j2"
+
+- name: Generate the required cron jobs for the node
+  local_action: "command python {{ role_path }}/files/fernet_rotate_cron_generator.py -t {{ (fernet_token_expiry | int) // 60 }} -i {{ groups['keystone'].index(inventory_hostname) }} -n {{ (groups['keystone'] | length) }}"
+  register: cron_jobs_json
+  when: keystone_token_provider == 'fernet'
+
+- name: Save the returned from cron jobs for building the crontab
+  set_fact:
+    cron_jobs: "{{ (cron_jobs_json.stdout | from_json).cron_jobs }}"
+  when: keystone_token_provider == 'fernet'
+
+- name: Copying files for keystone-fernet
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ node_config_directory }}/keystone-fernet/{{ item.dest }}"
+  with_items:
+    - { src: "crontab.j2", dest: "crontab" }
+    - { src: "fernet-rotate.sh.j2", dest: "fernet-rotate.sh" }
+    - { src: "fernet-node-sync.sh.j2", dest: "fernet-node-sync.sh" }
+  when: keystone_token_provider == 'fernet'
+
+- name: Copying files for keystone-ssh
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ node_config_directory }}/keystone-ssh/{{ item.dest }}"
+  with_items:
+    - { src: "sshd_config.j2", dest: "sshd_config" }
+    - { src: "id_rsa", dest: "id_rsa" }
+    - { src: "id_rsa.pub", dest: "id_rsa.pub" }
+    - { src: "ssh_config.j2", dest: "ssh_config" }
+  when: keystone_token_provider == 'fernet'
diff --git a/ansible/roles/keystone/tasks/deploy.yml b/ansible/roles/keystone/tasks/deploy.yml
index 10a7a1bf5..9ccf17b9a 100644
--- a/ansible/roles/keystone/tasks/deploy.yml
+++ b/ansible/roles/keystone/tasks/deploy.yml
@@ -8,6 +8,11 @@
 - include: start.yml
   when: inventory_hostname in groups['keystone']
 
+- include: init_fernet.yml
+  when:
+    - inventory_hostname in groups['keystone']
+    - keystone_token_provider == 'fernet'
+
 - include: register.yml
   when: inventory_hostname in groups['keystone']
 
diff --git a/ansible/roles/keystone/tasks/do_reconfigure.yml b/ansible/roles/keystone/tasks/do_reconfigure.yml
index a12287532..2eb6fdba7 100644
--- a/ansible/roles/keystone/tasks/do_reconfigure.yml
+++ b/ansible/roles/keystone/tasks/do_reconfigure.yml
@@ -1,4 +1,17 @@
 ---
+- name: Set variable for keystone components used in reconfigure
+  set_fact:
+    keystone_items:
+      - { name: keystone, group: keystone }
+
+- name: Add fernet related components to variable if fernet is enabled
+  set_fact:
+    keystone_fernet_items:
+      - { name: keystone_fernet, group: keystone }
+      - { name: keystone_ssh, group: keystone }
+    keystone_items: "{{ keystone_items + keystone_fernet_items }}"
+  when: keystone_token_provider == 'fernet'
+
 - name: Ensuring the containers up
   kolla_docker:
     name: "{{ item.name }}"
@@ -6,8 +19,7 @@
   register: container_state
   failed_when: container_state.Running == false
   when: inventory_hostname in groups[item.group]
-  with_items:
-    - { name: keystone, group: keystone }
+  with_items: keystone_items
 
 - include: config.yml
 
@@ -17,8 +29,7 @@
   failed_when: false
   register: check_results
   when: inventory_hostname in groups[item.group]
-  with_items:
-    - { name: keystone, group: keystone }
+  with_items: keystone_items
 
 # NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS'
 # and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE',
@@ -29,8 +40,7 @@
     action: "get_container_env"
   register: container_envs
   when: inventory_hostname in groups[item.group]
-  with_items:
-    - { name: keystone, group: keystone }
+  with_items: keystone_items
 
 - name: Remove the containers
   kolla_docker:
@@ -42,7 +52,7 @@
     - item[2]['rc'] == 1
     - inventory_hostname in groups[item[0]['group']]
   with_together:
-    - [{ name: keystone, group: keystone }]
+    - [keystone_items]
     - "{{ container_envs.results }}"
     - "{{ check_results.results }}"
 
@@ -59,6 +69,6 @@
     - item[2]['rc'] == 1
     - inventory_hostname in groups[item[0]['group']]
   with_together:
-    - [{ name: keystone, group: keystone }]
+    - [keystone_items]
     - "{{ container_envs.results }}"
     - "{{ check_results.results }}"
diff --git a/ansible/roles/keystone/tasks/init_fernet.yml b/ansible/roles/keystone/tasks/init_fernet.yml
new file mode 100644
index 000000000..0d2bb0076
--- /dev/null
+++ b/ansible/roles/keystone/tasks/init_fernet.yml
@@ -0,0 +1,15 @@
+---
+- name: Initialise fernet key authentication
+  command: "docker exec -t keystone_fernet kolla_keystone_bootstrap {{ keystone_username }} {{ keystone_groupname }}"
+  register: fernet_create
+  changed_when: "{{ fernet_create.stdout.find('localhost | SUCCESS => ') != -1 and (fernet_create.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: "(fernet_create.stdout.split()[2] == 'SUCCESS') or (fernet_create.stdout.find('Key repository is already initialized') != -1)"
+  retries: 10
+  delay: 5
+  run_once: True
+  delegate_to: "{{ groups['keystone'][0] }}"
+
+- name: Run key distribution
+  command: docker exec -t keystone_fernet /usr/bin/fernet-rotate.sh
+  run_once: True
+  delegate_to: "{{ groups['keystone'][0] }}"
\ No newline at end of file
diff --git a/ansible/roles/keystone/tasks/pull.yml b/ansible/roles/keystone/tasks/pull.yml
index b8c067236..544940070 100644
--- a/ansible/roles/keystone/tasks/pull.yml
+++ b/ansible/roles/keystone/tasks/pull.yml
@@ -5,3 +5,21 @@
     common_options: "{{ docker_common_options }}"
     image: "{{ keystone_image_full }}"
   when: inventory_hostname in groups['keystone']
+
+- name: Pulling keystone_fernet image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_fernet_image_full }}"
+  when:
+    - inventory_hostname in groups['keystone']
+    - keystone_token_provider == 'fernet'
+
+- name: Pulling keystone_ssh image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_ssh_image_full }}"
+  when:
+    - inventory_hostname in groups['keystone']
+    - keystone_token_provider == 'fernet'
\ No newline at end of file
diff --git a/ansible/roles/keystone/tasks/start.yml b/ansible/roles/keystone/tasks/start.yml
index 3108d1292..0b797d441 100644
--- a/ansible/roles/keystone/tasks/start.yml
+++ b/ansible/roles/keystone/tasks/start.yml
@@ -1,14 +1,49 @@
 ---
+- name: Set variable for inital keystone volumes
+  set_fact:
+    keystone_volumes:
+      - "{{ node_config_directory }}/keystone/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+
+- name: Add fernet volume to keystone volumes variable if fernet enabled
+  set_fact:
+    keystone_volumes: "{{ keystone_volumes + [\"keystone_fernet_tokens:/etc/keystone/fernet-keys\"] }}"
+  when: keystone_token_provider == 'fernet'
+
 - name: Starting keystone container
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
     image: "{{ keystone_image_full }}"
     name: "keystone"
+    volumes: "{{ keystone_volumes }}"
+
+- name: Wait for keystone startup
+  wait_for: host={{ kolla_internal_fqdn }} port={{ keystone_admin_port }}
+
+- name: Starting keystone-ssh container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_ssh_image_full }}"
+    name: "keystone_ssh"
     volumes:
-      - "{{ node_config_directory }}/keystone/:{{ container_config_directory }}/:ro"
+      - "{{ node_config_directory }}/keystone-ssh/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+      - "keystone_fernet_tokens:/etc/keystone/fernet-keys"
+  when: keystone_token_provider == 'fernet'
 
-- name: Wait for keystone startup
-  wait_for: host={{ kolla_internal_fqdn }} port={{ keystone_admin_port }}
+- name: Starting keystone-fernet container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_fernet_image_full }}"
+    name: "keystone_fernet"
+    volumes:
+      - "{{ node_config_directory }}/keystone-fernet/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "keystone_fernet_tokens:/etc/keystone/fernet-keys"
+  when: keystone_token_provider == 'fernet'
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/crontab.j2 b/ansible/roles/keystone/templates/crontab.j2
new file mode 100644
index 000000000..967309793
--- /dev/null
+++ b/ansible/roles/keystone/templates/crontab.j2
@@ -0,0 +1,3 @@
+{% for cron_job in cron_jobs %}
+{{ cron_job['min'] }} {{ cron_job['hour'] }} * * {{ cron_job['day'] }} /usr/bin/fernet-rotate.sh
+{% endfor %}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 b/ansible/roles/keystone/templates/fernet-node-sync.sh.j2
new file mode 100644
index 000000000..ffbd7c7dd
--- /dev/null
+++ b/ansible/roles/keystone/templates/fernet-node-sync.sh.j2
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Get data on the fernet tokens
+TOKEN_CHECK=$(/usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n {{ (groups['keystone'] | length) + 1 }})
+
+# Ensure the primary token exists and is not stale
+if $(echo "$TOKEN_CHECK" | grep -q '"update_required":"false"'); then
+    exit 0;
+fi
+
+# For each host node sync tokens
+{% for host in groups['keystone'] %}
+{% if inventory_hostname != host %}
+/usr/bin/rsync -azu --delete -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ keystone_ssh_port }}' keystone@{{ host }}:/etc/keystone/fernet-keys/ /etc/keystone/fernet-keys
+{% endif %}
+{% endfor %}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/fernet-rotate.sh.j2 b/ansible/roles/keystone/templates/fernet-rotate.sh.j2
new file mode 100644
index 000000000..e79b8909d
--- /dev/null
+++ b/ansible/roles/keystone/templates/fernet-rotate.sh.j2
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+keystone-manage --config-file /etc/keystone/keystone.conf fernet_rotate --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }}
+
+{% for host in groups['keystone'] %}
+{% if inventory_hostname != host %}
+/usr/bin/rsync -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ keystone_ssh_port }}' --delete /etc/keystone/fernet-keys/ keystone@{{ host }}:/etc/keystone/fernet-keys
+{% endif %}
+{% endfor %}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/id_rsa b/ansible/roles/keystone/templates/id_rsa
new file mode 100644
index 000000000..bdce5093e
--- /dev/null
+++ b/ansible/roles/keystone/templates/id_rsa
@@ -0,0 +1 @@
+{{ keystone_ssh_key.private_key }}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/id_rsa.pub b/ansible/roles/keystone/templates/id_rsa.pub
new file mode 100644
index 000000000..907b0e7e7
--- /dev/null
+++ b/ansible/roles/keystone/templates/id_rsa.pub
@@ -0,0 +1 @@
+{{ keystone_ssh_key.public_key }}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/keystone-fernet.json.j2 b/ansible/roles/keystone/templates/keystone-fernet.json.j2
new file mode 100644
index 000000000..3066d2ef6
--- /dev/null
+++ b/ansible/roles/keystone/templates/keystone-fernet.json.j2
@@ -0,0 +1,23 @@
+{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
+{
+    "command": "{{ cron_cmd }}",
+    "config_files": [{
+            "source": "{{ container_config_directory }}/crontab",
+            "dest": "/var/spool/cron/crontabs/root/fernet-cron",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/fernet-rotate.sh",
+            "dest": "/usr/bin/fernet-rotate.sh",
+            "owner": "root",
+            "perm": "0755"
+        },
+        {
+            "source": "{{ container_config_directory }}/fernet-node-sync.sh",
+            "dest": "/usr/bin/fernet-node-sync.sh",
+            "owner": "root",
+            "perm": "0755"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/keystone-ssh.json.j2 b/ansible/roles/keystone/templates/keystone-ssh.json.j2
new file mode 100644
index 000000000..c38fd6d62
--- /dev/null
+++ b/ansible/roles/keystone/templates/keystone-ssh.json.j2
@@ -0,0 +1,29 @@
+{
+    "command": "/usr/sbin/sshd -D",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/sshd_config",
+            "dest": "/etc/ssh/sshd_config",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/ssh_config",
+            "dest": "/var/lib/keystone/.ssh/config",
+            "owner": "keystone",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/id_rsa",
+            "dest": "/var/lib/keystone/.ssh/id_rsa",
+            "owner": "keystone",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/id_rsa.pub",
+            "dest": "/var/lib/keystone/.ssh/authorized_keys",
+            "owner": "keystone",
+            "perm": "0600"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/keystone.conf.j2 b/ansible/roles/keystone/templates/keystone.conf.j2
index 6e8bbf450..fa1013369 100644
--- a/ansible/roles/keystone/templates/keystone.conf.j2
+++ b/ansible/roles/keystone/templates/keystone.conf.j2
@@ -16,6 +16,15 @@ domain_specific_drivers_enabled = true
 domain_config_dir = /etc/keystone/domains
 {% endif %}
 
+{% if keystone_token_provider == 'fernet' %}
+[token]
+provider = {{ keystone_token_provider }}
+expiration = {{ fernet_token_expiry }}
+
+[fernet_tokens]
+max_active_keys = {{ (groups['keystone'] | length) + 1 }}
+{% endif %}
+
 [cache]
 backend = oslo_cache.memcache_pool
 enabled = True
diff --git a/ansible/roles/keystone/templates/ssh_config.j2 b/ansible/roles/keystone/templates/ssh_config.j2
new file mode 100644
index 000000000..f30dee26d
--- /dev/null
+++ b/ansible/roles/keystone/templates/ssh_config.j2
@@ -0,0 +1,4 @@
+Host {% for host in groups['keystone'] %}{% if inventory_hostname != host %}{{ host }} {% endif %}{% endfor %}
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
+  Port {{ keystone_ssh_port }}
\ No newline at end of file
diff --git a/ansible/roles/keystone/templates/sshd_config.j2 b/ansible/roles/keystone/templates/sshd_config.j2
new file mode 100644
index 000000000..8ccb34062
--- /dev/null
+++ b/ansible/roles/keystone/templates/sshd_config.j2
@@ -0,0 +1,5 @@
+Port {{ keystone_ssh_port }}
+ListenAddress {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+
+SyslogFacility AUTHPRIV
+UsePAM yes
\ No newline at end of file
diff --git a/ansible/roles/prechecks/tasks/service_checks.yml b/ansible/roles/prechecks/tasks/service_checks.yml
index 42335ab39..b275fd4c2 100644
--- a/ansible/roles/prechecks/tasks/service_checks.yml
+++ b/ansible/roles/prechecks/tasks/service_checks.yml
@@ -42,3 +42,9 @@
   register: result
   changed_when: false
   failed_when: result.stdout | regex_replace('(.*ssh_key.*)', '') | search(":")
+
+- name: Checking fernet_token_expiry in globals.yml. Update fernet_token_expiry to allowed value if this task fails
+  local_action: command grep '^[^#]*fernet_token_expiry:\s*\d*' "{{ CONFIG_DIR }}/globals.yml" | sed 's/[^0-9]*//g'
+  register: result
+  changed_when: false
+  failed_when: result.stdout | regex_replace('(60|120|180|240|300|360|600|720|900|1200|1800|3600|7200|10800|14400|21600|28800|43200|86400|604800)', '') | search(".+")
diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml
index 7bede69d8..e6d37bd0e 100644
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@@ -105,6 +105,15 @@ neutron_external_interface: "eth1"
 # Valid options are [ novnc, spice ]
 #nova_console: "novnc"
 
+# Valid options are [ uuid, fernet ]
+#keystone_token_provider: 'uuid'
+# Interval to rotate fernet keys by (in seconds). Must be an interval of
+# 60(1 min), 120(2 min), 180(3 min), 240(4 min), 300(5 min), 360(6 min),
+# 600(10 min), 720(12 min), 900(15 min), 1200(20 min), 1800(30 min),
+# 3600(1 hour), 7200(2 hour), 10800(3 hour), 14400(4 hour), 21600(6 hour),
+# 28800(8 hour), 43200(12 hour), 86400(1 day), 604800(1 week).
+#fernet_token_expiry: 86400
+
 # OpenStack services can be enabled or disabled with these options
 #enable_ceilometer: "no"
 #enable_central_logging: "no"
diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml
index 18e0b9dc1..c837663f9 100644
--- a/etc/kolla/passwords.yml
+++ b/etc/kolla/passwords.yml
@@ -81,6 +81,10 @@ kolla_ssh_key:
   private_key:
   public_key:
 
+keystone_ssh_key:
+  private_key:
+  public_key:
+
 ####################
 # RabbitMQ options
 ####################
diff --git a/kolla/cmd/genpwd.py b/kolla/cmd/genpwd.py
index b45fa56ad..5ea950d16 100755
--- a/kolla/cmd/genpwd.py
+++ b/kolla/cmd/genpwd.py
@@ -43,7 +43,7 @@ def main():
     uuid_keys = ['ceph_cluster_fsid', 'rbd_secret_uuid']
 
     # SSH key pair
-    ssh_keys = ['kolla_ssh_key', 'nova_ssh_key']
+    ssh_keys = ['kolla_ssh_key', 'nova_ssh_key', 'keystone_ssh_key']
 
     # If these keys are None, leave them as None
     blank_keys = ['docker_registry_password']
diff --git a/releasenotes/notes/add-fernet-support-54ccb88b901d8d8b.yaml b/releasenotes/notes/add-fernet-support-54ccb88b901d8d8b.yaml
new file mode 100644
index 000000000..e765fde01
--- /dev/null
+++ b/releasenotes/notes/add-fernet-support-54ccb88b901d8d8b.yaml
@@ -0,0 +1,3 @@
+---
+features:
+  - Add full support for fernet with distributed token node syncing