blob: ecd3a10ac11248465dbe152e3607583f65e79a94 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#!/bin/bash
set -o xtrace
set -o errexit
# Enable unbuffered output for Ansible in Jenkins.
export PYTHONUNBUFFERED=1
# neutron_agents_wrappers (see ansible/roles/neutron) spawns per-namespace
# helper containers (dnsmasq/haproxy/keepalived) that are not managed by
# Kolla and can be created/removed by Neutron independently of the deploy,
# e.g. neutron_dhcp_agent_dnsmasq_qdhcp-<uuid>. Ignore them in these checks.
IGNORE_CONTAINERS_REGEX='_(qdhcp|qrouter|ovnmeta)-'
filter_unmanaged_containers() {
grep -Ev "$IGNORE_CONTAINERS_REGEX" || true
}
check_podman_failures() {
failed_containers=$(sudo podman ps -a --format "{{.Names}}" \
--filter status=created \
--filter status=exited \
--filter status=paused \
--filter status=removing \
--filter status=unknown \
| filter_unmanaged_containers)
for container in $failed_containers; do
sudo podman inspect $container
sudo podman logs $container
done
}
check_podman_unhealthies() {
while [ -n "$(sudo podman ps -a --format "{{.Names}}" --filter health=starting)" ]; do
echo "Containers with health status 'starting', waiting..."
sleep 10
done
unhealthy_containers=$(sudo podman ps -a --format "{{.Names}}" \
--filter health=unhealthy \
| filter_unmanaged_containers)
for container in $unhealthy_containers; do
echo "Discovered unhealthy container: $container"
echo "$container - podman inspect"
sudo podman inspect $container
echo "$container - ps axwuf"
sudo podman exec $container ps axwuf
echo "$container - ss -anp"
sudo podman exec $container ss -anp
done
}
check_docker_failures() {
# All docker container's status are created, restarting, running, removing,
# paused, exited and dead. Containers without running status are treated as
# failure. removing is added in docker 1.13, just ignore it now.
# In addition to that, containers in unhealthy state (from healthchecks)
# are treated as failure.
failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
--filter status=created \
--filter status=exited \
--filter status=dead \
--filter status=paused \
--filter status=removing \
--filter status=restarting \
| filter_unmanaged_containers)
for container in $failed_containers; do
sudo docker inspect $container
sudo docker logs $container
done
}
check_docker_unhealthies() {
while [ -n "$(sudo docker ps -a --format "{{.Names}}" --filter health=starting)" ]; do
echo "Containers with health status 'starting', waiting..."
sleep 10
done
unhealthy_containers=$(sudo docker ps -a --format "{{.Names}}" \
--filter health=unhealthy \
| filter_unmanaged_containers)
for container in $unhealthy_containers; do
echo "Discovered unhealthy container: $container"
echo "$container - docker inspect"
sudo docker inspect $container
echo "$container - ps axwuf"
sudo docker exec $container ps axwuf
echo "$container - ss -anp"
sudo docker exec $container ss -anp
done
}
check_failure() {
if [ "$CONTAINER_ENGINE" = "docker" ]; then
check_docker_failures
check_docker_unhealthies
elif [ "$CONTAINER_ENGINE" = "podman" ]; then
check_podman_failures
check_podman_unhealthies
else
echo "Invalid container engine: ${CONTAINER_ENGINE}"
exit 1
fi
if [[ -n "$unhealthy_containers" ]]; then
# NOTE(mnasiadka): try checking them again, because usually they are healthy now
echo "Discovered unhealthy containers - sleeping 60 seconds and retrying check"
sleep 60
if [ "$CONTAINER_ENGINE" = "docker" ]; then
check_docker_unhealthies
elif [ "$CONTAINER_ENGINE" = "podman" ]; then
check_podman_unhealthies
fi
# NOTE(mnasiadka): If they're unhealthy again - let's fail
if [[ -n "$unhealthy_containers" ]]; then
exit 1;
fi
fi
if [[ -n "$failed_containers" ]]; then
exit 1;
fi
}
check_failure
|
