blob: 19d2f071f2854820d49de61f6db79477be8f453e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
#!/bin/bash
set -o xtrace
set -o errexit
# Enable unbuffered output for Ansible in Jenkins.
export PYTHONUNBUFFERED=1
check_podman_failures() {
failed_containers=$(sudo podman ps -a --format "{{.Names}}" \
--filter status=created \
--filter status=paused \
--filter status=exited \
--filter status=unknown)
for container in $failed_containers; do
sudo podman inspect $container
sudo podman logs $container
done
}
check_podman_unhealthies() {
while [ -n "$(sudo podman ps -a --format "{{.Names}}" --filter health=starting)" ]; do
echo "Containers with health status 'starting', waiting..."
sleep 10
done
unhealthy_containers=$(sudo podman ps -a --format "{{.Names}}" \
--filter health=unhealthy)
for container in $unhealthy_containers; do
echo "Discovered unhealthy container: $container"
echo "$container - podman inspect"
sudo podman inspect $container
echo "$container - ps axwuf"
sudo podman exec $container ps axwuf
echo "$container - ss -anp"
sudo podman exec $container ss -anp
done
}
check_docker_failures() {
# All docker container's status are created, restarting, running, removing,
# paused, exited and dead. Containers without running status are treated as
# failure. removing is added in docker 1.13, just ignore it now.
# In addition to that, containers in unhealthy state (from healthchecks)
# are treated as failure.
failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
--filter status=created \
--filter status=restarting \
--filter status=paused \
--filter status=exited \
--filter status=dead)
for container in $failed_containers; do
sudo docker inspect $container
sudo docker logs $container
done
}
check_docker_unhealthies() {
while [ -n "$(sudo docker ps -a --format "{{.Names}}" --filter health=starting)" ]; do
echo "Containers with health status 'starting', waiting..."
sleep 10
done
unhealthy_containers=$(sudo docker ps -a --format "{{.Names}}" \
--filter health=unhealthy)
for container in $unhealthy_containers; do
echo "Discovered unhealthy container: $container"
echo "$container - docker inspect"
sudo docker inspect $container
echo "$container - ps axwuf"
sudo docker exec $container ps axwuf
echo "$container - ss -anp"
sudo docker exec $container ss -anp
done
}
check_failure() {
if [ "$CONTAINER_ENGINE" = "docker" ]; then
check_docker_failures
check_docker_unhealthies
elif [ "$CONTAINER_ENGINE" = "podman" ]; then
check_podman_failures
check_podman_unhealthies
else
echo "Invalid container engine: ${CONTAINER_ENGINE}"
exit 1
fi
if [[ -n "$unhealthy_containers" ]]; then
# NOTE(mnasiadka): try checking them again, because usually they are healthy now
echo "Discovered unhealthy containers - sleeping 60 seconds and retrying check"
sleep 60
if [ "$CONTAINER_ENGINE" = "docker" ]; then
check_docker_unhealthies
elif [ "$CONTAINER_ENGINE" = "podman" ]; then
check_podman_unhealthies
fi
# NOTE(mnasiadka): If they're unhealthy again - let's fail
if [[ -n "$unhealthy_containers" ]]; then
exit 1;
fi
fi
if [[ -n "$failed_containers" ]]; then
exit 1;
fi
}
check_failure
|
