Improve rebuild feature: handle check upgrade status changes, run rebuild via cron and add a deploy cron job
This commit is contained in:
parent
2b07eab7e2
commit
525473a7fc
2 changed files with 479 additions and 269 deletions
29
README.md
29
README.md
|
|
@ -25,16 +25,25 @@ service nagios-nrpe-server reload
|
|||
|
||||
```
|
||||
Usage : check_container_upgrade [-d] [-E /path/to/engine] [container1,...]
|
||||
-E [path] Force a specific engine (possible values: auto docker podman, default: auto)
|
||||
-x [container] Exclude specified container (could be repeat)
|
||||
-M [integer] Max number of container checks to run in parallel (default: 4, 0=no limit)
|
||||
-f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the docker-compose.yml file
|
||||
-b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker compose file if provided)
|
||||
--rebuild-lock Specify rebuild lock file path (default: /var/tmp/check_container_upgrade.lock)
|
||||
--rebuild-log Specify rebuild log file path (default: /var/tmp/check_container_upgrade.log)
|
||||
-d Debug mode
|
||||
-X Enable bash tracing (=set -x)
|
||||
-h Show this message
|
||||
-E [path] Force a specific engine (possible values: auto docker podman,
|
||||
default: auto)
|
||||
-x [container] Exclude specified container (could be repeat)
|
||||
-M [integer] Max number of container checks to run in parallel
|
||||
(default: 4, 0=no limit)
|
||||
-f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the
|
||||
docker-compose.yml file
|
||||
-b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker
|
||||
compose file if provided)
|
||||
--rebuild-path Specify rebuild data directory path (default: /var/log/check_container_upgrade)
|
||||
--rebuild-cron Start in rebuild cron mode: rebuild containers detected and mark to be
|
||||
rebuilt on status file.
|
||||
--deploy-cron Start in deploy cron mode: deploy containers known as rebuilt in status
|
||||
file.
|
||||
-d Debug mode
|
||||
-l Log file
|
||||
-C Console logging (even if log file is specify)
|
||||
-X Enable bash tracing (=set -x)
|
||||
-h Show this message
|
||||
```
|
||||
|
||||
## Copyright
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
# Monitoring plugin to check if running containers are upgradable
|
||||
#
|
||||
#
|
||||
# Author: Benjamin Renard <brenard@zionetrix.net>
|
||||
# Date: Sun, 03 Mar 2024 16:40:19 +0100
|
||||
# Source: https://gitea.zionetrix.net/bn8/check_container_upgrade
|
||||
|
|
@ -10,64 +10,154 @@ ENGINE="auto"
|
|||
POSSIBLE_ENGINES=( "auto" "docker" "podman" )
|
||||
DOCKERCOMPOSE_FILE=""
|
||||
DEBUG=0
|
||||
CONSOLE=0
|
||||
LOG_FILE=0
|
||||
MAX_PARALLEL_CHECKS=4
|
||||
ONLY_CONTAINERS=()
|
||||
EXCLUDED_CONTAINERS=( buildx_buildkit_default )
|
||||
REBUILD=0
|
||||
REBUILD_LOCK_FILE="/var/tmp/$( basename $0 ).lock"
|
||||
REBUILD_LOG_FILE="/var/tmp/$( basename $0 ).log"
|
||||
REBUILD_DATA_DIR="/var/log/$(basename $0)"
|
||||
REBUILD_CRON=0
|
||||
DEPLOY_CRON=0
|
||||
declare -rA CHECK_PLUGINS=(
|
||||
["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l"
|
||||
["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk"
|
||||
["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l"
|
||||
["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk"
|
||||
)
|
||||
|
||||
function now() { [[ -z "$1" ]] && date "+%F %H:%M:%S" || date -d "@$1" "+%F %H:%M:%S" ; }
|
||||
function current_time() { date "+%s"; }
|
||||
|
||||
function debug() {
|
||||
if [ $DEBUG -eq 1 ]
|
||||
then
|
||||
>&2 echo -e "[DEBUG] $@"
|
||||
fi
|
||||
[[ $DEBUG -eq 0 ]] && return
|
||||
if [[ -n "$LOG_FILE" ]] && [[ $CONSOLE -eq 1 ]]; then
|
||||
echo -e "$(now) - [DEBUG] $@" | tee -a "$LOG_FILE" 2>&1
|
||||
elif [[ -n "$LOG_FILE" ]]; then
|
||||
echo -e "$(now) - [DEBUG] $@" >> "$LOG_FILE"
|
||||
else
|
||||
>&2 echo -e "[DEBUG] $@"
|
||||
fi
|
||||
}
|
||||
|
||||
function log() {
|
||||
[[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${@:2}" >> "$LOG_FILE"
|
||||
[[ "$1" == "ERROR" ]] && >&2 echo -e "ERROR - ${@:2}" || echo -e "${@:2}"
|
||||
}
|
||||
|
||||
function message() { log INFO $@ ; }
|
||||
function error() { log ERROR $@ ; }
|
||||
|
||||
function is_empty() {
|
||||
[ $# -gt 0 ] && return 1
|
||||
return 0
|
||||
[ $# -gt 0 ] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
function in_array() {
|
||||
param=$1;
|
||||
shift;
|
||||
for elem in "$@";
|
||||
do
|
||||
[[ "$param" = "$elem" ]] && return 0;
|
||||
done;
|
||||
return 1
|
||||
param=$1;
|
||||
shift;
|
||||
for elem in "$@";
|
||||
do
|
||||
[[ "$param" = "$elem" ]] && return 0;
|
||||
done;
|
||||
return 1
|
||||
}
|
||||
|
||||
function implode() {
|
||||
local d=${1-} f=${2-}
|
||||
if shift 2; then
|
||||
printf %s "$f" "${@/#/$d}"
|
||||
fi
|
||||
local d=${1-} f=${2-}
|
||||
if shift 2; then
|
||||
printf %s "$f" "${@/#/$d}"
|
||||
fi
|
||||
}
|
||||
|
||||
function format_duration {
|
||||
local T=$1
|
||||
local D=$((T/60/60/24))
|
||||
local H=$((T/60/60%24))
|
||||
local M=$((T/60%60))
|
||||
local S=$((T%60))
|
||||
(( $D > 0 )) && printf '%d days and ' $D
|
||||
printf '%02d:%02d:%02d' $H $M $S
|
||||
}
|
||||
|
||||
REBUILD_STATUS_FILE=""
|
||||
function rebuild_status_file() {
|
||||
[[ ! -e "$REBUILD_DATA_DIR" ]] && mkdir -p "$REBUILD_DATA_DIR"
|
||||
[[ -z "$REBUILD_STATUS_FILE" ]] && \
|
||||
REBUILD_STATUS_FILE="$REBUILD_DATA_DIR/status.json"
|
||||
[[ -e "$REBUILD_STATUS_FILE" ]] || echo '{}' > "$REBUILD_STATUS_FILE"
|
||||
echo "$REBUILD_STATUS_FILE"
|
||||
}
|
||||
|
||||
function rebuild_status() {
|
||||
if [[ -z "$1" ]]; then
|
||||
cat "$(rebuild_status_file)"
|
||||
else
|
||||
rebuild_status | jq -r --arg container "$1" '.[$container]'
|
||||
fi
|
||||
}
|
||||
|
||||
function update_rebuild_status() {
|
||||
if [[ "$1" == "-d" ]]; then
|
||||
local data=$(
|
||||
rebuild_status | jq \
|
||||
--arg container "$2" \
|
||||
'del(.[$container])'
|
||||
)
|
||||
else
|
||||
local args=( --arg container "$1" )
|
||||
local expr=( )
|
||||
for arg in "${@:2}"; do
|
||||
local name=$( echo "$arg" | cut -d'=' -f1 )
|
||||
local value=$( echo "$arg" | sed 's/[^=]\+=//' )
|
||||
args+=( --arg "$name" "$value" )
|
||||
expr+=( ".[\$container].$name=\$$name" )
|
||||
done
|
||||
local data=$(
|
||||
rebuild_status | jq \
|
||||
"${args[@]}" "$( implode ' | ' "${expr[@]}" )"
|
||||
)
|
||||
fi
|
||||
cat <<< $data > $(rebuild_status_file)
|
||||
}
|
||||
|
||||
function remove_rebuild_status() {
|
||||
rebuild_status_file > /dev/null
|
||||
[[ -e "$REBUILD_STATUS_FILE" ]] || return 0
|
||||
debug "Remove previous rebuild status file ($REBUILD_STATUS_FILE) and log files it contains"
|
||||
for log in $( jq -r '.[] | .log' $REBUILD_STATUS_FILE ); do
|
||||
debug " remove old container log file $log"
|
||||
rm -f $log
|
||||
done
|
||||
debug " remove status file"
|
||||
rm -f "$REBUILD_STATUS_FILE"
|
||||
}
|
||||
|
||||
function usage() {
|
||||
error="$1"
|
||||
[ -n "$error" ] && echo "$error"
|
||||
cat << EOF
|
||||
error="$1"
|
||||
[ -n "$error" ] && message "$error"
|
||||
cat << EOF
|
||||
Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
|
||||
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]}, default: $ENGINE)
|
||||
-x [container] Exclude specified container (could be repeat)
|
||||
-M [integer] Max number of container checks to run in parallel (default: $MAX_PARALLEL_CHECKS, 0=no limit)
|
||||
-f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the docker-compose.yml file
|
||||
-b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker compose file if provided)
|
||||
--rebuild-lock Specify rebuild lock file path (default: ${REBUILD_LOCK_FILE})
|
||||
--rebuild-log Specify rebuild log file path (default: ${REBUILD_LOG_FILE})
|
||||
-d Debug mode
|
||||
-X Enable bash tracing (=set -x)
|
||||
-h Show this message
|
||||
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]},
|
||||
default: $ENGINE)
|
||||
-x [container] Exclude specified container (could be repeat)
|
||||
-M [integer] Max number of container checks to run in parallel
|
||||
(default: $MAX_PARALLEL_CHECKS, 0=no limit)
|
||||
-f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the
|
||||
docker-compose.yml file
|
||||
-b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker
|
||||
compose file if provided)
|
||||
--rebuild-path Specify rebuild data directory path (default: ${REBUILD_DATA_DIR})
|
||||
--rebuild-cron Start in rebuild cron mode: rebuild containers detected and mark to be
|
||||
rebuilt on status file.
|
||||
--deploy-cron Start in deploy cron mode: deploy containers known as rebuilt in status
|
||||
file.
|
||||
-d Debug mode
|
||||
-l Log file
|
||||
-C Console logging (even if log file is specify)
|
||||
-X Enable bash tracing (=set -x)
|
||||
-h Show this message
|
||||
EOF
|
||||
[ -n "$error" ] && exit 1
|
||||
exit 0
|
||||
[ -n "$error" ] && exit 1
|
||||
exit 0
|
||||
}
|
||||
|
||||
idx=1
|
||||
|
|
@ -75,82 +165,165 @@ while [ $idx -le $# ]
|
|||
do
|
||||
OPT=${!idx}
|
||||
case $OPT in
|
||||
-d)
|
||||
DEBUG=1
|
||||
;;
|
||||
-h)
|
||||
usage
|
||||
;;
|
||||
-E)
|
||||
((idx++))
|
||||
ENGINE=${!idx}
|
||||
if [ ! -x "$ENGINE" ]
|
||||
then
|
||||
in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE"
|
||||
fi
|
||||
;;
|
||||
-f)
|
||||
((idx++))
|
||||
DOCKERCOMPOSE_FILE=${!idx}
|
||||
;;
|
||||
-b|--build|--rebuild)
|
||||
REBUILD=1
|
||||
;;
|
||||
--rebuild-lock)
|
||||
((idx++))
|
||||
REBUILD_LOCK_FILE="${!idx}"
|
||||
;;
|
||||
--rebuild-log)
|
||||
((idx++))
|
||||
REBUILD_LOG_FILE="${!idx}"
|
||||
;;
|
||||
-x)
|
||||
((idx++))
|
||||
EXCLUDED_CONTAINERS+=( ${!idx} )
|
||||
;;
|
||||
-M)
|
||||
((idx++))
|
||||
MAX_PARALLEL_CHECKS=${!idx}
|
||||
;;
|
||||
-X)
|
||||
set -x
|
||||
;;
|
||||
*)
|
||||
ONLY_CONTAINERS+=( $OPT )
|
||||
;;
|
||||
-d)
|
||||
DEBUG=1
|
||||
;;
|
||||
-C)
|
||||
CONSOLE=1
|
||||
;;
|
||||
-l)
|
||||
((idx++))
|
||||
LOG_FILE=${!idx}
|
||||
;;
|
||||
-h)
|
||||
usage
|
||||
;;
|
||||
-E)
|
||||
((idx++))
|
||||
ENGINE=${!idx}
|
||||
if [ ! -x "$ENGINE" ]
|
||||
then
|
||||
in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE"
|
||||
fi
|
||||
;;
|
||||
-f)
|
||||
((idx++))
|
||||
DOCKERCOMPOSE_FILE=${!idx}
|
||||
;;
|
||||
-b|--build|--rebuild)
|
||||
REBUILD=1
|
||||
;;
|
||||
--rebuild-path)
|
||||
((idx++))
|
||||
REBUILD_DATA_DIR="${!idx}"
|
||||
;;
|
||||
-x)
|
||||
((idx++))
|
||||
EXCLUDED_CONTAINERS+=( ${!idx} )
|
||||
;;
|
||||
-M)
|
||||
((idx++))
|
||||
MAX_PARALLEL_CHECKS=${!idx}
|
||||
;;
|
||||
-X)
|
||||
set -x
|
||||
;;
|
||||
--rebuild-cron)
|
||||
REBUILD_CRON=1
|
||||
;;
|
||||
--deploy-cron)
|
||||
DEPLOY_CRON=1
|
||||
;;
|
||||
*)
|
||||
ONLY_CONTAINERS+=( $OPT )
|
||||
;;
|
||||
esac
|
||||
((idx++))
|
||||
done
|
||||
|
||||
debug "Start with parameters: $@"
|
||||
|
||||
! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}"
|
||||
|
||||
if [ "$ENGINE" == "auto" ]
|
||||
then
|
||||
debug "Auto-detect engine..."
|
||||
for engine in ${POSSIBLE_ENGINES[@]}
|
||||
do
|
||||
[ "$engine" == "auto" ] && continue
|
||||
which "$engine" > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
debug "$engine not found"
|
||||
continue
|
||||
fi
|
||||
ENGINE="$engine"
|
||||
break
|
||||
done
|
||||
if [ -z "$ENGINE" ]
|
||||
then
|
||||
echo "UNKNOWN - Fail to auto-detect engine"
|
||||
exit 3
|
||||
fi
|
||||
debug "Auto-detected engine: $ENGINE"
|
||||
debug "Auto-detect engine..."
|
||||
for engine in ${POSSIBLE_ENGINES[@]}
|
||||
do
|
||||
[ "$engine" == "auto" ] && continue
|
||||
which "$engine" > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
debug "$engine not found"
|
||||
continue
|
||||
fi
|
||||
ENGINE="$engine"
|
||||
break
|
||||
done
|
||||
if [ -z "$ENGINE" ]
|
||||
then
|
||||
message "UNKNOWN - Fail to auto-detect engine"
|
||||
exit 3
|
||||
fi
|
||||
debug "Auto-detected engine: $ENGINE"
|
||||
fi
|
||||
|
||||
if [ -n "$DOCKERCOMPOSE_FILE" -a ! -e "$DOCKERCOMPOSE_FILE" ]
|
||||
then
|
||||
echo "UNKNOWN - Docker compose file not found ($DOCKERCOMPOSE_FILE)"
|
||||
exit 3
|
||||
if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
|
||||
if [[ ! -e "$DOCKERCOMPOSE_FILE" ]]; then
|
||||
message "UNKNOWN - Docker compose file not found ($DOCKERCOMPOSE_FILE)"
|
||||
exit 3
|
||||
fi
|
||||
which "${ENGINE}-compose" > /dev/null 2>&1 && \
|
||||
COMPOSE_BIN="${ENGINE}-compose" || \
|
||||
COMPOSE_BIN="$ENGINE compose"
|
||||
debug "Docker compose bin: $COMPOSE_BIN"
|
||||
fi
|
||||
|
||||
if [[ $REBUILD_CRON -eq 1 ]]; then
|
||||
to_rebuild=(
|
||||
$(
|
||||
rebuild_status | jq -r -c \
|
||||
'to_entries[] | select((.value.start_date|not) and (.value.error|not)) | .key'
|
||||
)
|
||||
)
|
||||
if [[ ${#to_rebuild[@]} -eq 0 ]]; then
|
||||
debug "No container need to be rebuild"
|
||||
exit 0
|
||||
fi
|
||||
message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[@]}"
|
||||
error=0
|
||||
for container in ${to_rebuild[@]}; do
|
||||
log="$REBUILD_DATA_DIR/$container.log"
|
||||
message " $container: start building image (log=$log)"
|
||||
start_time=$(current_time)
|
||||
update_rebuild_status "$container" "start_date=$(now $start_time)" "log=$log"
|
||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache $container >> $log 2>&1
|
||||
result=$?
|
||||
end_time=$(current_time)
|
||||
(( duration=end_time-start_time ))
|
||||
duration=$(format_duration $duration)
|
||||
container_info=( "end_date=$(now $end_time)" "duration=$duration" )
|
||||
if [[ $result -eq 0 ]]; then
|
||||
message " $container: rebuilt in $duration"
|
||||
else
|
||||
error " $container: fail to rebuild image"
|
||||
container_info+=( "error=fail to rebuild image" )
|
||||
fi
|
||||
update_rebuild_status "$container" "${container_info[@]}"
|
||||
done
|
||||
message "No more container to rebuild, stop"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ $DEPLOY_CRON -eq 1 ]]; then
|
||||
to_deploy=(
|
||||
$(
|
||||
rebuild_status | jq -r -c \
|
||||
'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key'
|
||||
)
|
||||
)
|
||||
if [[ ${#to_deploy[@]} -eq 0 ]]; then
|
||||
debug "No container need to be deploy"
|
||||
exit 0
|
||||
fi
|
||||
message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[@]}"
|
||||
error=0
|
||||
for container in ${to_deploy[@]}; do
|
||||
message " $container: deploying..."
|
||||
log="$REBUILD_DATA_DIR/$container.log"
|
||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps $container > $log 2>&1
|
||||
if [[ $? -eq 0 ]]; then
|
||||
message " $container: done"
|
||||
update_rebuild_status -d "$container"
|
||||
rm -f $log
|
||||
else
|
||||
error " $container: fail to deploy new container image"
|
||||
update_rebuild_status "$container" "error=fail to deploy new container image"
|
||||
error=1
|
||||
fi
|
||||
done
|
||||
message "done"
|
||||
exit $error
|
||||
fi
|
||||
|
||||
EXIT_CODE=0
|
||||
|
|
@ -159,219 +332,247 @@ declare -A CONTAINER_PID
|
|||
declare -A UP_TO_DATE
|
||||
declare -A ERRORS
|
||||
declare -A UNKNOWNS
|
||||
UPGRADABLE_CONTAINERS=( )
|
||||
declare -A UPGRADABLE_CONTAINERS
|
||||
CHECKED_CONTAINERS=( )
|
||||
|
||||
debug "List running containers..."
|
||||
if [ -n "$DOCKERCOMPOSE_FILE" ]
|
||||
then
|
||||
which "${ENGINE}-compose" > /dev/null 2>&1 \
|
||||
&& COMPOSE_BIN="${ENGINE}-compose" \
|
||||
|| COMPOSE_BIN="$ENGINE compose"
|
||||
RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ')
|
||||
else
|
||||
RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ')
|
||||
fi
|
||||
[[ -n "$DOCKERCOMPOSE_FILE" ]] && \
|
||||
RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ') ||
|
||||
RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ')
|
||||
debug "Running containers: $RUNNING_CONTAINERS"
|
||||
|
||||
function exec_in_container() {
|
||||
container=$1
|
||||
shift;
|
||||
if [ -n "$DOCKERCOMPOSE_FILE" ]
|
||||
then
|
||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE exec $container $@
|
||||
return $?
|
||||
fi
|
||||
$ENGINE exec $container $@
|
||||
return $?
|
||||
container=$1
|
||||
shift;
|
||||
if [ -n "$DOCKERCOMPOSE_FILE" ]
|
||||
then
|
||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE exec $container $@
|
||||
return $?
|
||||
fi
|
||||
$ENGINE exec $container $@
|
||||
return $?
|
||||
}
|
||||
|
||||
# Implement check inside a function to allow running it in parallel
|
||||
# Parameters : [container] [output file]
|
||||
function check_container() {
|
||||
container="$1"
|
||||
output_file="$2"
|
||||
STATUS=""
|
||||
for check_plugin in ${CHECK_PLUGINS[@]}
|
||||
do
|
||||
exec_in_container $container test -e $check_plugin > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
debug "$container - Plugin $check_plugin not found"
|
||||
continue
|
||||
fi
|
||||
debug "$container - Plugin $check_plugin found, use it"
|
||||
STATUS="$(exec_in_container $container ${CHECK_PLUGINS[${check_plugin}]} 2>&1)"
|
||||
ex=$?
|
||||
debug "$container - Plugin output: $STATUS"
|
||||
debug "$container - Plugin exit code: $ex"
|
||||
break
|
||||
done
|
||||
if [ -z "$STATUS" ]
|
||||
then
|
||||
debug "$container - No check plugin found"
|
||||
STATUS="UNKNOWN - No check plugin available"
|
||||
ex=3
|
||||
fi
|
||||
echo $STATUS > $output_file
|
||||
return $ex
|
||||
container="$1"
|
||||
output_file="$2"
|
||||
STATUS=""
|
||||
for check_plugin in ${CHECK_PLUGINS[@]}
|
||||
do
|
||||
exec_in_container $container test -e $check_plugin > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
debug "$container - Plugin $check_plugin not found"
|
||||
continue
|
||||
fi
|
||||
debug "$container - Plugin $check_plugin found, use it"
|
||||
STATUS="$(exec_in_container $container ${CHECK_PLUGINS[${check_plugin}]} 2>&1)"
|
||||
ex=$?
|
||||
debug "$container - Plugin output: $STATUS"
|
||||
debug "$container - Plugin exit code: $ex"
|
||||
break
|
||||
done
|
||||
if [ -z "$STATUS" ]
|
||||
then
|
||||
debug "$container - No check plugin found"
|
||||
STATUS="UNKNOWN - No check plugin available"
|
||||
ex=3
|
||||
fi
|
||||
echo $STATUS > $output_file
|
||||
return $ex
|
||||
}
|
||||
|
||||
debug "Trigger check of all selected containers..."
|
||||
for container in $RUNNING_CONTAINERS
|
||||
do
|
||||
if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]}
|
||||
then
|
||||
debug "$container - Ignored"
|
||||
continue
|
||||
fi
|
||||
if in_array $container ${EXCLUDED_CONTAINERS[@]}
|
||||
then
|
||||
debug "$container - Excluded"
|
||||
continue
|
||||
fi
|
||||
if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ]
|
||||
then
|
||||
debug "Max parallel checks count reached. Waiting some check ending"
|
||||
wait -n
|
||||
debug "Some check ended, continue"
|
||||
fi
|
||||
CHECKED_CONTAINERS+=( "$container" )
|
||||
CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) )
|
||||
check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! )
|
||||
if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]}
|
||||
then
|
||||
debug "$container - Ignored"
|
||||
continue
|
||||
fi
|
||||
if in_array $container ${EXCLUDED_CONTAINERS[@]}
|
||||
then
|
||||
debug "$container - Excluded"
|
||||
continue
|
||||
fi
|
||||
if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ]
|
||||
then
|
||||
debug "Max parallel checks count reached. Waiting some check ending"
|
||||
wait -n
|
||||
debug "Some check ended, continue"
|
||||
fi
|
||||
CHECKED_CONTAINERS+=( "$container" )
|
||||
CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) )
|
||||
check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! )
|
||||
done
|
||||
|
||||
debug "Wait for each individual container check and handle their result..."
|
||||
for container in ${!CONTAINER_PID[@]}
|
||||
do
|
||||
pid=${CONTAINER_PID[$container]}
|
||||
debug "$container - Waiting for PID ${pid}..."
|
||||
wait $pid
|
||||
ex=$?
|
||||
debug "$container - Check return ${ex}"
|
||||
STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} )
|
||||
rm -f ${CONTAINER_STATUS_FILE[$container]}
|
||||
if [ $ex -eq 0 ]
|
||||
then
|
||||
UP_TO_DATE+=( ["$container"]=$STATUS )
|
||||
else
|
||||
ERRORS+=( ["$container"]=$STATUS )
|
||||
[ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || UPGRADABLE_CONTAINERS+=( "$container" )
|
||||
fi
|
||||
[ $EXIT_CODE -ge $ex ] && continue
|
||||
[ $ex -gt 3 ] && ex=3
|
||||
EXIT_CODE=$ex
|
||||
pid=${CONTAINER_PID[$container]}
|
||||
debug "$container - Waiting for PID ${pid}..."
|
||||
wait $pid
|
||||
ex=$?
|
||||
debug "$container - Check return ${ex}"
|
||||
STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} )
|
||||
rm -f ${CONTAINER_STATUS_FILE[$container]}
|
||||
if [ $ex -eq 0 ]
|
||||
then
|
||||
UP_TO_DATE+=( ["$container"]=$STATUS )
|
||||
else
|
||||
ERRORS+=( ["$container"]=$STATUS )
|
||||
[ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || \
|
||||
UPGRADABLE_CONTAINERS+=( ["$container"]="$STATUS" )
|
||||
fi
|
||||
[ $EXIT_CODE -ge $ex ] && continue
|
||||
[ $ex -gt 3 ] && ex=3
|
||||
EXIT_CODE=$ex
|
||||
done
|
||||
|
||||
NOTFOUNDS=()
|
||||
if ! is_empty $ONLY_CONTAINERS
|
||||
then
|
||||
for container in ${ONLY_CONTAINERS[@]}
|
||||
do
|
||||
if ! in_array $container ${CHECKED_CONTAINERS[@]}
|
||||
then
|
||||
debug "$container - Container not found"
|
||||
ERRORS+=( ["$container"]="Container not found" )
|
||||
NOTFOUNDS+=( "$container" )
|
||||
EXIT_CODE=3
|
||||
fi
|
||||
done
|
||||
for container in ${ONLY_CONTAINERS[@]}
|
||||
do
|
||||
if ! in_array $container ${CHECKED_CONTAINERS[@]}
|
||||
then
|
||||
debug "$container - Container not found"
|
||||
ERRORS+=( ["$container"]="Container not found" )
|
||||
NOTFOUNDS+=( "$container" )
|
||||
EXIT_CODE=3
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
debug "Final exit code: $EXIT_CODE"
|
||||
|
||||
debug "Check containers (${#CHECKED_CONTAINERS[@]}): $( implode ", " "${CHECKED_CONTAINERS[@]}" )"
|
||||
debug "Up-to-date containers (${#UP_TO_DATE[@]}): $( implode ", " "${!UP_TO_DATE[@]}" )"
|
||||
debug "Upgradable containers (${#UPGRADABLE_CONTAINERS[@]}): $( implode ", " "${UPGRADABLE_CONTAINERS[@]}" )"
|
||||
debug "Upgradable containers (${#UPGRADABLE_CONTAINERS[@]}): $( implode ", " "${!UPGRADABLE_CONTAINERS[@]}" )"
|
||||
debug "Containers with errors (${#ERRORS[@]}): $( implode ", " "${!ERRORS[@]}" )"
|
||||
debug "Not found containers (${#NOTFOUNDS[@]}): $( implode ", " "${NOTFOUNDS[@]}" )"
|
||||
|
||||
# Compute performance data
|
||||
let CONTAINER_COUNTS=${#CHECKED_CONTAINERS[@]}+${#NOTFOUNDS[@]}
|
||||
PERF_DATA=(
|
||||
"uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS"
|
||||
"upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS"
|
||||
"containers_with_errors=${#ERRORS[@]};1;;0;$CONTAINER_COUNTS"
|
||||
"unknown_state_containers=${#UNKNOWNS[@]};;;0;$CONTAINER_COUNTS"
|
||||
"uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS"
|
||||
"upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS"
|
||||
"containers_with_errors=${#ERRORS[@]};1;;0;$CONTAINER_COUNTS"
|
||||
"unknown_state_containers=${#UNKNOWNS[@]};;;0;$CONTAINER_COUNTS"
|
||||
)
|
||||
|
||||
# Compute performance data as string
|
||||
PERF_DATA_TXT="$( implode " " "${PERF_DATA[@]}" )"
|
||||
|
||||
# Display check result message
|
||||
case $EXIT_CODE in
|
||||
0)
|
||||
echo -n "OK - All ${#UP_TO_DATE[@]} container(s) are up-to-date"
|
||||
;;
|
||||
1)
|
||||
echo -n "WARNING - ${#ERRORS[@]} container(s) need to be updated"
|
||||
;;
|
||||
2)
|
||||
echo -n "CRITICAL - ${#ERRORS[@]} container(s) need to be updated"
|
||||
;;
|
||||
*)
|
||||
echo -n "UNKNOWN - fail to retrieve status of ${#UNKNOWNS[@]} container(s)"
|
||||
;;
|
||||
0)
|
||||
message "OK - All ${#UP_TO_DATE[@]} container(s) are up-to-date |$PERF_DATA_TXT"
|
||||
;;
|
||||
1)
|
||||
message "WARNING - ${#ERRORS[@]} container(s) need to be updated |$PERF_DATA_TXT"
|
||||
;;
|
||||
2)
|
||||
message "CRITICAL - ${#ERRORS[@]} container(s) need to be updated |$PERF_DATA_TXT"
|
||||
;;
|
||||
*)
|
||||
message "UNKNOWN - fail to retrieve status of ${#UNKNOWNS[@]} container(s) |$PERF_DATA_TXT"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Add performance data
|
||||
echo " |$( implode " " "${PERF_DATA[@]}" )"
|
||||
|
||||
# Trigger container build (if need, enabled and docker compose file is provided)
|
||||
if [ $REBUILD -eq 1 ]
|
||||
then
|
||||
if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]
|
||||
then
|
||||
debug "No upgradable container to rebuild"
|
||||
if [ -e "$REBUILD_LOCK_FILE" ]
|
||||
then
|
||||
debug "Remove previous rebuild lock file ($REBUILD_LOCK_FILE)"
|
||||
rm -f "$REBUILD_LOCK_FILE"
|
||||
fi
|
||||
elif [ -z "$DOCKERCOMPOSE_FILE" ]
|
||||
then
|
||||
echo
|
||||
echo "WARNING: No docker compose file provided, can't trigger rebuild of following container(s):"
|
||||
echo "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )"
|
||||
elif [ -e "$REBUILD_LOCK_FILE" ]
|
||||
then
|
||||
REBUILD_PID=$(cat $REBUILD_LOCK_FILE)
|
||||
echo
|
||||
if [ -d "/proc/$REBUILD_PID" ]
|
||||
then
|
||||
echo "Rebuild already triggered on $( stat --format %y "$REBUILD_LOCK_FILE" ) (PID: $REBUILD_PID)"
|
||||
echo "You could follow it in its log file: ${REBUILD_LOG_FILE}"
|
||||
else
|
||||
echo "Container images already rebuilt (see ${REBUILD_LOG_FILE} for details)."
|
||||
echo "You could recreate and restart them using the following command:"
|
||||
echo
|
||||
echo " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${UPGRADABLE_CONTAINERS[@]}"
|
||||
echo
|
||||
echo "Note:"
|
||||
echo " If new updates was published since rebuild was triggered, remove the"
|
||||
echo " '$REBUILD_LOCK_FILE' file and recheck it to trigger a new build."
|
||||
fi
|
||||
echo
|
||||
else
|
||||
echo
|
||||
echo "Trigger rebuild of following container(s): ${UPGRADABLE_CONTAINERS[@]}"
|
||||
echo "You could follow the rebuild process in its log file: ${REBUILD_LOG_FILE}"
|
||||
echo "You will able to recreate and restart them using the following command:"
|
||||
echo
|
||||
echo " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${UPGRADABLE_CONTAINERS[@]}"
|
||||
echo
|
||||
debug "Check if we have to trigger some rebuild (status file: $(rebuild_status_file))"
|
||||
if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]
|
||||
then
|
||||
debug "No upgradable container to rebuild"
|
||||
remove_rebuild_status
|
||||
elif [ -z "$DOCKERCOMPOSE_FILE" ]
|
||||
then
|
||||
message
|
||||
message "WARNING: No docker compose file provided, can't trigger rebuild of following" \
|
||||
"container(s):"
|
||||
message "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )"
|
||||
else
|
||||
message "Rebuilding containers:"
|
||||
REBUILT_CONTAINERS=()
|
||||
for container in ${!UPGRADABLE_CONTAINERS[@]}; do
|
||||
need_rebuild=0
|
||||
container_data=$( rebuild_status "$container" )
|
||||
if [[ "$container_data" != "null" ]]; then
|
||||
debug "$container: data=$container_data"
|
||||
trigger_date=$( jq -r .trigger_date <<< $container_data )
|
||||
start_date=$( jq -r .start_date <<< $container_data )
|
||||
end_date=$( jq -r .end_date <<< $container_data )
|
||||
log=$( jq -r .log <<< $container_data )
|
||||
if [[ "$start_date" == "null" ]]; then
|
||||
debug "$container: build triggered but not yet started"
|
||||
message "- $container: rebuild triggered on $trigger_date and not yet started"
|
||||
elif [[ "$end_date" == "null" ]]; then
|
||||
debug "$container: rebuild triggered on $trigger_date and started on" \
|
||||
"$start_date, but not yet finish"
|
||||
message "- $container: rebuild triggered on $trigger_date and started on" \
|
||||
"$start_date, but not yet finish (log: $log)"
|
||||
else
|
||||
duration=$( jq -r .duration <<< $container_data )
|
||||
debug "$container: rebuilt in $duration on $start_date (finish on $end_date)"
|
||||
prev_status=$( jq -r .status <<< $container_data )
|
||||
if [[ "$prev_status" == "${UPGRADABLE_CONTAINERS[$container]}" ]]; then
|
||||
error=$( jq -r .error <<< $container_data )
|
||||
if [[ "$error" != "null" ]]; then
|
||||
message "- $container: $error (log: $log)"
|
||||
else
|
||||
message "- $container: already rebuilt in $duration (rebuild" \
|
||||
"triggered on $trigger_date, started on $start_date and finish" \
|
||||
"at $end_date,log: $log)"
|
||||
REBUILT_CONTAINERS+=( $container )
|
||||
fi
|
||||
else
|
||||
update_rebuild_status -d "$container"
|
||||
message "- $container: upgrade status change since last rebuild, rebuild" \
|
||||
"it again"
|
||||
need_rebuild=1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
debug "$container: not found in status file"
|
||||
need_rebuild=1
|
||||
fi
|
||||
if [[ $need_rebuild -eq 1 ]]; then
|
||||
update_rebuild_status "$container" \
|
||||
"trigger_date=$(now)" "status=${UPGRADABLE_CONTAINERS[$container]}"
|
||||
message "- $container: rebuild triggered"
|
||||
else
|
||||
debug "$container: rebuild not need"
|
||||
fi
|
||||
done
|
||||
|
||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache ${UPGRADABLE_CONTAINERS[@]} > $REBUILD_LOG_FILE 2>&1 &
|
||||
echo $! > "$REBUILD_LOCK_FILE"
|
||||
fi
|
||||
# Handle rebuilt containers
|
||||
if [[ ${#REBUILT_CONTAINERS[@]} -gt 0 ]]; then
|
||||
message
|
||||
message "Some containers are ready to be recreated and restarted."
|
||||
message "Run the following command to do it:"
|
||||
message
|
||||
message " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[@]}"
|
||||
fi
|
||||
message
|
||||
fi
|
||||
fi
|
||||
|
||||
# Display details, starting by errors
|
||||
for container in ${!ERRORS[@]}
|
||||
do
|
||||
echo ${container} - ${ERRORS[${container}]}
|
||||
message ${container} - ${ERRORS[${container}]}
|
||||
done
|
||||
|
||||
for container in ${!UP_TO_DATE[@]}
|
||||
do
|
||||
echo ${container} - ${UP_TO_DATE[${container}]}
|
||||
message ${container} - ${UP_TO_DATE[${container}]}
|
||||
done
|
||||
|
||||
exit $EXIT_CODE
|
||||
|
||||
# vim: shiftwidth=4 tabstop=4 expandtab
|
||||
|
|
|
|||
Loading…
Reference in a new issue