Improve performance by running container checks in parallel
This commit is contained in:
parent
6d4437efb6
commit
54fdabfaa7
2 changed files with 75 additions and 30 deletions
|
@ -22,6 +22,7 @@ service nagios-nrpe-server reload
|
||||||
Usage : check_container_upgrade [-d] [-E /path/to/engine] [container1,...]
|
Usage : check_container_upgrade [-d] [-E /path/to/engine] [container1,...]
|
||||||
-E [path] Force a specific engine (possible values: auto docker podman, default: auto)
|
-E [path] Force a specific engine (possible values: auto docker podman, default: auto)
|
||||||
-x [container] Exclude specified container (could be repeat)
|
-x [container] Exclude specified container (could be repeat)
|
||||||
|
-M [integer] Max number of container checks to run in parallel (default: 4, 0=no limit)
|
||||||
-d Debug mode
|
-d Debug mode
|
||||||
-X Enable bash tracing (=set -x)
|
-X Enable bash tracing (=set -x)
|
||||||
-h Show this message
|
-h Show this message
|
||||||
|
|
|
@ -9,6 +9,8 @@
|
||||||
ENGINE="auto"
|
ENGINE="auto"
|
||||||
POSSIBLE_ENGINES=( "auto" "docker" "podman" )
|
POSSIBLE_ENGINES=( "auto" "docker" "podman" )
|
||||||
DEBUG=0
|
DEBUG=0
|
||||||
|
MAX_PARALLEL_CHECKS=4
|
||||||
|
ONLY_CONTAINERS=()
|
||||||
EXCLUDED_CONTAINERS=( buildx_buildkit_default )
|
EXCLUDED_CONTAINERS=( buildx_buildkit_default )
|
||||||
declare -rA CHECK_PLUGINS=(
|
declare -rA CHECK_PLUGINS=(
|
||||||
["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l"
|
["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l"
|
||||||
|
@ -18,7 +20,7 @@ declare -rA CHECK_PLUGINS=(
|
||||||
function debug() {
|
function debug() {
|
||||||
if [ $DEBUG -eq 1 ]
|
if [ $DEBUG -eq 1 ]
|
||||||
then
|
then
|
||||||
>&2 echo -e "[DEBUG] $1"
|
>&2 echo -e "[DEBUG] $@"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,9 +43,10 @@ function usage() {
|
||||||
error="$1"
|
error="$1"
|
||||||
[ -n "$error" ] && echo "$error"
|
[ -n "$error" ] && echo "$error"
|
||||||
cat << EOF
|
cat << EOF
|
||||||
Usage : $0 [-d] [-E /path/to/engine] [container1,...]
|
Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
|
||||||
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]}, default: $ENGINE)
|
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]}, default: $ENGINE)
|
||||||
-x [container] Exclude specified container (could be repeat)
|
-x [container] Exclude specified container (could be repeat)
|
||||||
|
-M [integer] Max number of container checks to run in parallel (default: $MAX_PARALLEL_CHECKS, 0=no limit)
|
||||||
-d Debug mode
|
-d Debug mode
|
||||||
-X Enable bash tracing (=set -x)
|
-X Enable bash tracing (=set -x)
|
||||||
-h Show this message
|
-h Show this message
|
||||||
|
@ -52,34 +55,43 @@ EOF
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
||||||
while getopts "hdXE:x:" OPTION
|
idx=1
|
||||||
|
while [ $idx -le $# ]
|
||||||
do
|
do
|
||||||
case $OPTION in
|
OPT=${!idx}
|
||||||
d)
|
case $OPT in
|
||||||
|
-d)
|
||||||
DEBUG=1
|
DEBUG=1
|
||||||
;;
|
;;
|
||||||
h)
|
-h)
|
||||||
usage
|
usage
|
||||||
;;
|
;;
|
||||||
E)
|
-E)
|
||||||
ENGINE=$OPTARG
|
((idx++))
|
||||||
|
ENGINE=${!idx}
|
||||||
if [ ! -x "$ENGINE" ]
|
if [ ! -x "$ENGINE" ]
|
||||||
then
|
then
|
||||||
in_array $ENGINE $POSSIBLE_ENGINES || usage "Invalid engine $ENGINE"
|
in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE"
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
x)
|
-x)
|
||||||
EXCLUDED_CONTAINERS+=( $OPTARG )
|
((idx++))
|
||||||
|
EXCLUDED_CONTAINERS+=( ${!idx} )
|
||||||
;;
|
;;
|
||||||
X)
|
-M)
|
||||||
|
((idx++))
|
||||||
|
MAX_PARALLEL_CHECKS=${!idx}
|
||||||
|
;;
|
||||||
|
-X)
|
||||||
set -x
|
set -x
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
usage "Unknown option $OPTION"
|
ONLY_CONTAINERS+=( $OPT )
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
((idx++))
|
||||||
done
|
done
|
||||||
ONLY_CONTAINERS=( "${@:$OPTIND}" )
|
|
||||||
! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}"
|
! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}"
|
||||||
|
|
||||||
if [ "$ENGINE" == "auto" ]
|
if [ "$ENGINE" == "auto" ]
|
||||||
|
@ -106,6 +118,8 @@ then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
EXIT_CODE=0
|
EXIT_CODE=0
|
||||||
|
declare -A CONTAINER_STATUS_FILE
|
||||||
|
declare -A CONTAINER_PID
|
||||||
declare -A UPTODATE
|
declare -A UPTODATE
|
||||||
declare -A ERRORS
|
declare -A ERRORS
|
||||||
CHECKED_CONTAINERS=( )
|
CHECKED_CONTAINERS=( )
|
||||||
|
@ -114,19 +128,11 @@ debug "List running containers..."
|
||||||
RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ')
|
RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ')
|
||||||
debug "Running containers: $RUNNING_CONTAINERS"
|
debug "Running containers: $RUNNING_CONTAINERS"
|
||||||
|
|
||||||
for container in $RUNNING_CONTAINERS
|
# Implement check inside a function to allow running it in parallel
|
||||||
do
|
# Parameters : [container] [output file]
|
||||||
if ! is_empty $ONLY_CONTAINERS && ! in_array $container $ONLY_CONTAINERS
|
function check_container() {
|
||||||
then
|
container="$1"
|
||||||
debug "$container - Ignored"
|
output_file="$2"
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if in_array $container $EXCLUDED_CONTAINERS
|
|
||||||
then
|
|
||||||
debug "$container - Excluded"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
CHECKED_CONTAINERS+=( "$container" )
|
|
||||||
STATUS=""
|
STATUS=""
|
||||||
for check_plugin in ${CHECK_PLUGINS[@]}
|
for check_plugin in ${CHECK_PLUGINS[@]}
|
||||||
do
|
do
|
||||||
|
@ -149,6 +155,44 @@ do
|
||||||
STATUS="UNKNOWN - No check plugin available"
|
STATUS="UNKNOWN - No check plugin available"
|
||||||
ex=3
|
ex=3
|
||||||
fi
|
fi
|
||||||
|
echo $STATUS > $output_file
|
||||||
|
return $ex
|
||||||
|
}
|
||||||
|
|
||||||
|
debug "Trigger check of all selected containers..."
|
||||||
|
for container in $RUNNING_CONTAINERS
|
||||||
|
do
|
||||||
|
if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]}
|
||||||
|
then
|
||||||
|
debug "$container - Ignored"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if in_array $container ${EXCLUDED_CONTAINERS[@]}
|
||||||
|
then
|
||||||
|
debug "$container - Excluded"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ]
|
||||||
|
then
|
||||||
|
debug "Max parallel checks count reached. Waiting some check ending"
|
||||||
|
wait -n
|
||||||
|
debug "Some check ended, continue"
|
||||||
|
fi
|
||||||
|
CHECKED_CONTAINERS+=( "$container" )
|
||||||
|
CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) )
|
||||||
|
check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! )
|
||||||
|
done
|
||||||
|
|
||||||
|
debug "Wait for each individual container check and handle their result..."
|
||||||
|
for container in ${!CONTAINER_PID[@]}
|
||||||
|
do
|
||||||
|
pid=${CONTAINER_PID[$container]}
|
||||||
|
debug "$container - Waiting for PID ${pid}..."
|
||||||
|
wait $pid
|
||||||
|
ex=$?
|
||||||
|
debug "$container - Check return ${ex}"
|
||||||
|
STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} )
|
||||||
|
rm -f ${CONTAINER_STATUS_FILE[$container]}
|
||||||
if [ $ex -eq 0 ]
|
if [ $ex -eq 0 ]
|
||||||
then
|
then
|
||||||
UPTODATE+=( ["$container"]=$STATUS )
|
UPTODATE+=( ["$container"]=$STATUS )
|
||||||
|
@ -164,7 +208,7 @@ if ! is_empty $ONLY_CONTAINERS
|
||||||
then
|
then
|
||||||
for container in ${ONLY_CONTAINERS[@]}
|
for container in ${ONLY_CONTAINERS[@]}
|
||||||
do
|
do
|
||||||
if ! in_array $container $CHECKED_CONTAINERS
|
if ! in_array $container ${CHECKED_CONTAINERS[@]}
|
||||||
then
|
then
|
||||||
debug "$container - Not found"
|
debug "$container - Not found"
|
||||||
ERRORS+=( ["$container"]="Not found" )
|
ERRORS+=( ["$container"]="Not found" )
|
||||||
|
|
Loading…
Reference in a new issue