Merge "CI: add prometheus-efk scenario"

3 files changed, 167 insertions, 1 deletions

diff --git a/tests/run.yml b/tests/run.yml
index c538cd6a2..a7f0be9fe 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -10,7 +10,7 @@
     - name: set facts for commonly used variables
       vars:
         # NOTE(yoctozepto): needed here to use in other facts too
-        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb'] }}"
+        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk'] }}"
       set_fact:
         kolla_inventory_path: "/etc/kolla/inventory"
         logs_dir: "/tmp/logs"
@@ -405,6 +405,14 @@
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "mariadb"
+
+        - name: Run test-prometheus-efk.sh script
+          script:
+            cmd: test-prometheus-efk.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: scenario == "prometheus-efk"
+
       when: scenario != "bifrost"
 
 # NOTE(yoctozepto): each host checks itself
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index 4d41ec3c4..31d96bc61 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -137,3 +137,11 @@ neutron_plugin_agent: "linuxbridge"
 {% if scenario == "ovn" %}
 neutron_plugin_agent: "ovn"
 {% endif %}
+
+{% if scenario == "prometheus-efk" %}
+enable_chrony: "no"
+enable_central_logging: "yes"
+enable_grafana: "yes"
+enable_prometheus: "yes"
+enable_prometheus_openstack_exporter: "no"
+{% endif %}
diff --git a/tests/test-prometheus-efk.sh b/tests/test-prometheus-efk.sh
new file mode 100755
index 000000000..1828ea610
--- /dev/null
+++ b/tests/test-prometheus-efk.sh
@@ -0,0 +1,150 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+
+# Enable unbuffered output
+export PYTHONUNBUFFERED=1
+
+function check_kibana {
+    # Query kibana, and check that the returned page looks like a kibana page.
+    KIBANA_URL=${OS_AUTH_URL%:*}:5601
+    output_path=$1
+    kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml)
+    args=(
+        --include
+        --location
+        --fail
+        --user
+        kibana:$kibana_password
+    )
+    if [[ "$TLS_ENABLED" = "True" ]]; then
+        args+=(--cacert $OS_CACERT)
+    fi
+    if ! curl "${args[@]}" $KIBANA_URL > $output_path; then
+        return 1
+    fi
+    if ! grep '<title>Kibana</title>' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function check_grafana {
+    # Query grafana, and check that the returned page looks like a grafana page.
+    GRAFANA_URL=${OS_AUTH_URL%:*}:3000
+    output_path=$1
+    grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml)
+    args=(
+        --include
+        --location
+        --fail
+        --user
+        admin:$grafana_password
+    )
+    if [[ "$TLS_ENABLED" = "True" ]]; then
+        args+=(--cacert $OS_CACERT)
+    fi
+    if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then
+        return 1
+    fi
+    if ! grep '<title>Grafana</title>' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function check_prometheus {
+    # Query prometheus graph, and check that the returned page looks like a
+    # prometheus page.
+    PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph
+    output_path=$1
+    args=(
+        --include
+        --location
+        --fail
+    )
+    if [[ "$TLS_ENABLED" = "True" ]]; then
+        args+=(--cacert $OS_CACERT)
+    fi
+    if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then
+        return 1
+    fi
+    if ! grep '<title>Prometheus' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function test_kibana {
+    # TODO(mgoddard): Query elasticsearch for logs.
+    echo "TESTING: Kibana"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_kibana $output_path; do
+        echo "Kibana not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Kibana did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Kibana"
+}
+
+function test_grafana {
+    echo "TESTING: Grafana"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_grafana $output_path; do
+        echo "Grafana not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Grafana did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Grafana"
+}
+
+function test_prometheus {
+    # TODO(mgoddard): Query metrics.
+    echo "TESTING: Prometheus"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_prometheus $output_path; do
+        echo "Prometheus not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Prometheus did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Prometheus"
+}
+
+function test_prometheus_efk_logged {
+    . /etc/kolla/admin-openrc.sh
+
+    test_kibana
+    test_grafana
+    test_prometheus
+}
+
+function test_prometheus_efk {
+    echo "Testing prometheus and EFK"
+    test_prometheus_efk_logged > /tmp/logs/ansible/test-prometheus-efk 2>&1
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Testing prometheus and EFK failed. See ansible/test-prometheus-efk for details"
+    else
+        echo "Successfully tested prometheus and EFK. See ansible/test-prometheus-efk for details"
+    fi
+    return $result
+}
+
+test_prometheus_efk