Compare commits

...

2 commits

3 changed files with 179 additions and 169 deletions

15
.editorconfig Normal file
View file

@ -0,0 +1,15 @@
root = true
[*]
indent_style = space
indent_size = 4
trim_trailing_whitespace = true
insert_final_newline = true
charset = utf-8
end_of_line = lf
[*.{yaml,yml}]
indent_size = 2
[Makefile]
indent_style = tab

View file

@ -13,13 +13,17 @@ repos:
- --ignore-regex=.*codespell-ignore$
#- --write-changes # Uncomment to write changes
exclude_types: [csv, json]
- repo: https://github.com/adrienverge/yamllint
rev: v1.32.0
hooks:
- id: yamllint
ignore: .github/
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.7.1
hooks:
- id: prettier
args: ["--print-width", "100"]
- repo: https://github.com/adrienverge/yamllint
rev: v1.32.0
hooks:
- id: yamllint
ignore: .github/
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.10.0.1
hooks:
- id: shellcheck

View file

@ -16,7 +16,7 @@ MAX_PARALLEL_CHECKS=4
ONLY_CONTAINERS=()
EXCLUDED_CONTAINERS=( buildx_buildkit_default )
REBUILD=0
REBUILD_DATA_DIR="/var/log/$(basename $0)"
REBUILD_DATA_DIR="/var/log/$(basename "$0")"
REBUILD_CRON=0
DEPLOY_CRON=0
declare -rA CHECK_PLUGINS=(
@ -24,38 +24,43 @@ declare -rA CHECK_PLUGINS=(
["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk"
)
function now() { [[ -z "$1" ]] && date "+%F %H:%M:%S" || date -d "@$1" "+%F %H:%M:%S" ; }
function now() {
if [[ -z "$1" ]]; then date "+%F %H:%M:%S"; else date -d "@$1" "+%F %H:%M:%S"; fi
}
function current_time() { date "+%s"; }
function debug() {
[[ $DEBUG -eq 0 ]] && return
if [[ -n "$LOG_FILE" ]] && [[ $CONSOLE -eq 1 ]]; then
echo -e "$(now) - [DEBUG] $@" | tee -a "$LOG_FILE" 2>&1
echo -e "$(now) - [DEBUG] $*" | tee -a "$LOG_FILE" 2>&1
elif [[ -n "$LOG_FILE" ]]; then
echo -e "$(now) - [DEBUG] $@" >> "$LOG_FILE"
echo -e "$(now) - [DEBUG] $*" >> "$LOG_FILE"
else
>&2 echo -e "[DEBUG] $@"
>&2 echo -e "[DEBUG] $*"
fi
}
function log() {
[[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${@:2}" >> "$LOG_FILE"
[[ "$1" == "ERROR" ]] && >&2 echo -e "ERROR - ${@:2}" || echo -e "${@:2}"
[[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${*:2}" >> "$LOG_FILE"
if [[ "$1" == "ERROR" ]]; then
>&2 echo -e "ERROR - ${*:2}"
else
echo -e "${*:2}"
fi
}
function message() { log INFO $@ ; }
function error() { log ERROR $@ ; }
function message() { log INFO "$@" ; }
function error() { log ERROR "$@" ; }
function is_empty() {
[ $# -gt 0 ] && return 1
[[ $# -gt 0 ]] && return 1
return 0
}
function in_array() {
param=$1;
local param=$1 elem;
shift;
for elem in "$@";
do
for elem in "$@"; do
[[ "$param" = "$elem" ]] && return 0;
done;
return 1
@ -69,13 +74,13 @@ function implode() {
}
function format_duration {
local T=$1
local D=$((T/60/60/24))
local H=$((T/60/60%24))
local M=$((T/60%60))
local S=$((T%60))
(( $D > 0 )) && printf '%d days and ' $D
printf '%02d:%02d:%02d' $H $M $S
local t=$1
local d=$((t/60/60/24))
local h=$((t/60/60%24))
local m=$((t/60%60))
local s=$((t%60))
[[ $d -gt 0 ]] && printf '%d days and ' $d
printf '%02d:%02d:%02d' $h $m $s
}
REBUILD_STATUS_FILE=""
@ -84,58 +89,64 @@ function rebuild_status_file() {
[[ -z "$REBUILD_STATUS_FILE" ]] && \
REBUILD_STATUS_FILE="$REBUILD_DATA_DIR/status.json"
[[ -e "$REBUILD_STATUS_FILE" ]] || echo '{}' > "$REBUILD_STATUS_FILE"
echo "$REBUILD_STATUS_FILE"
}
function rebuild_status() {
if [[ -z "$1" ]]; then
cat "$(rebuild_status_file)"
local output_var=$1
[[ -n "$REBUILD_STATUS_FILE" ]] || rebuild_status_file
if [[ -z "$2" ]]; then
declare -g "$output_var=$( cat "$REBUILD_STATUS_FILE" )"
else
rebuild_status | jq -r --arg container "$1" '.[$container]'
declare -g "$output_var=$(
jq -r --arg container "$2" '.[$container]' < "$REBUILD_STATUS_FILE"
)"
fi
}
function update_rebuild_status() {
local data
if [[ "$1" == "-d" ]]; then
local data=$(
rebuild_status | jq \
rebuild_status DATA
# shellcheck disable=SC2153
data=$(
jq \
--arg container "$2" \
'del(.[$container])'
'del(.[$container])' <<< "$DATA"
)
else
local args=( --arg container "$1" )
local expr=( )
local args=( --arg container "$1" ) arg name value exp=( )
for arg in "${@:2}"; do
local name=$( echo "$arg" | cut -d'=' -f1 )
local value=$( echo "$arg" | sed 's/[^=]\+=//' )
name=$( head -n 1 <<< "$arg" | cut -d'=' -f1 )
# shellcheck disable=SC2001
value=$( sed "s/$name=//" <<< "$arg" )
args+=( --arg "$name" "$value" )
expr+=( ".[\$container].$name=\$$name" )
exp+=( ".[\$container].${name}=\$${name}" )
done
local data=$(
rebuild_status | jq \
"${args[@]}" "$( implode ' | ' "${expr[@]}" )"
rebuild_status DATA
data=$(
jq "${args[@]}" "$( implode ' | ' "${exp[@]}" )" <<< "$DATA"
)
fi
cat <<< $data > $(rebuild_status_file)
cat <<< "$data" > "$REBUILD_STATUS_FILE"
}
function remove_rebuild_status() {
rebuild_status_file > /dev/null
[[ -n "$REBUILD_STATUS_FILE" ]] || rebuild_status_file
[[ -e "$REBUILD_STATUS_FILE" ]] || return 0
debug "Remove previous rebuild status file ($REBUILD_STATUS_FILE) and log files it contains"
for log in $( jq -r '.[] | .log' $REBUILD_STATUS_FILE ); do
for log in $( jq -r '.[] | .log' "$REBUILD_STATUS_FILE" ); do
debug " remove old container log file $log"
rm -f $log
rm -f "$log"
done
debug " remove status file"
rm -f "$REBUILD_STATUS_FILE"
}
function usage() {
error="$1"
[ -n "$error" ] && message "$error"
local error="$1"
[[ -n "$error" ]] && message "$error"
cat << EOF
Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
Usage : $(basename "$0") [-d] [-E /path/to/engine] [container1,...]
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]},
default: $ENGINE)
-x [container] Exclude specified container (could be repeat)
@ -156,13 +167,12 @@ Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
-X Enable bash tracing (=set -x)
-h Show this message
EOF
[ -n "$error" ] && exit 1
[[ -n "$error" ]] && exit 1
exit 0
}
idx=1
while [ $idx -le $# ]
do
while [[ $idx -le $# ]]; do
OPT=${!idx}
case $OPT in
-d)
@ -181,9 +191,8 @@ do
-E)
((idx++))
ENGINE=${!idx}
if [ ! -x "$ENGINE" ]
then
in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE"
if [[ ! -x "$ENGINE" ]]; then
in_array "$ENGINE" "${POSSIBLE_ENGINES[@]}" || usage "Invalid engine $ENGINE"
fi
;;
-f)
@ -199,7 +208,7 @@ do
;;
-x)
((idx++))
EXCLUDED_CONTAINERS+=( ${!idx} )
EXCLUDED_CONTAINERS+=( "${!idx}" )
;;
-M)
((idx++))
@ -215,33 +224,27 @@ do
DEPLOY_CRON=1
;;
*)
ONLY_CONTAINERS+=( $OPT )
ONLY_CONTAINERS+=( "$OPT" )
;;
esac
((idx++))
done
debug "Start with parameters: $@"
debug "Start with parameters: $*"
! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}"
is_empty "${ONLY_CONTAINERS[@]}" || debug "Only containers: ${ONLY_CONTAINERS[*]}"
if [ "$ENGINE" == "auto" ]
then
if [[ "$ENGINE" == "auto" ]]; then
debug "Auto-detect engine..."
for engine in ${POSSIBLE_ENGINES[@]}
do
[ "$engine" == "auto" ] && continue
which "$engine" > /dev/null 2>&1
if [ $? -ne 0 ]
then
debug "$engine not found"
continue
fi
for engine in "${POSSIBLE_ENGINES[@]}"; do
[[ "$engine" == "auto" ]] && continue
if which "$engine" > /dev/null 2>&1; then
ENGINE="$engine"
break
fi
debug "$engine not found"
done
if [ -z "$ENGINE" ]
then
if [[ -z "$ENGINE" ]]; then
message "UNKNOWN - Fail to auto-detect engine"
exit 3
fi
@ -260,29 +263,29 @@ if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
fi
if [[ $REBUILD_CRON -eq 1 ]]; then
to_rebuild=(
$(
rebuild_status | jq -r -c \
'to_entries[] | select((.value.start_date|not) and (.value.error|not)) | .key'
)
rebuild_status DATA
mapfile -t to_rebuild < <(
jq -r -c 'to_entries[] | select(
(.value.start_date|not) and (.value.error|not)
) | .key' <<< "$DATA"
)
if [[ ${#to_rebuild[@]} -eq 0 ]]; then
debug "No container need to be rebuild"
exit 0
fi
message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[@]}"
message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[*]}"
error=0
for container in ${to_rebuild[@]}; do
for container in "${to_rebuild[@]}"; do
log="$REBUILD_DATA_DIR/$container.log"
message " $container: start building image (log=$log)"
start_time=$(current_time)
update_rebuild_status "$container" "start_date=$(now $start_time)" "log=$log"
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache $container >> $log 2>&1
update_rebuild_status "$container" "start_date=$(now "$start_time")" "log=$log"
"$COMPOSE_BIN" -f "$DOCKERCOMPOSE_FILE" build --no-cache "$container" >> "$log" 2>&1
result=$?
end_time=$(current_time)
(( duration=end_time-start_time ))
duration=$(format_duration $duration)
container_info=( "end_date=$(now $end_time)" "duration=$duration" )
duration=$(format_duration "$duration")
container_info=( "end_date=$(now "$end_time")" "duration=$duration" )
if [[ $result -eq 0 ]]; then
message " $container: rebuilt in $duration"
else
@ -296,26 +299,24 @@ if [[ $REBUILD_CRON -eq 1 ]]; then
fi
if [[ $DEPLOY_CRON -eq 1 ]]; then
to_deploy=(
$(
rebuild_status | jq -r -c \
'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key'
)
rebuild_status DATA
mapfile -t to_deploy < <(
jq -r -c \
'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key' <<< "$DATA"
)
if [[ ${#to_deploy[@]} -eq 0 ]]; then
debug "No container need to be deploy"
exit 0
fi
message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[@]}"
message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[*]}"
error=0
for container in ${to_deploy[@]}; do
for container in "${to_deploy[@]}"; do
message " $container: deploying..."
log="$REBUILD_DATA_DIR/$container.log"
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps $container > $log 2>&1
if [[ $? -eq 0 ]]; then
if $COMPOSE_BIN -f "$DOCKERCOMPOSE_FILE" up -d --no-deps "$container" > "$log" 2>&1; then
message " $container: done"
update_rebuild_status -d "$container"
rm -f $log
rm -f "$log"
else
error " $container: fail to deploy new container image"
update_rebuild_status "$container" "error=fail to deploy new container image"
@ -331,113 +332,106 @@ declare -A CONTAINER_STATUS_FILE
declare -A CONTAINER_PID
declare -A UP_TO_DATE
declare -A ERRORS
declare -A UNKNOWNS
declare -a UNKNOWNS
declare -A UPGRADABLE_CONTAINERS
CHECKED_CONTAINERS=( )
debug "List running containers..."
[[ -n "$DOCKERCOMPOSE_FILE" ]] && \
RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ') ||
if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
RUNNING_CONTAINERS=$(
$COMPOSE_BIN -f "$DOCKERCOMPOSE_FILE" ps --format '{{.Service}}' | tr '\n' ' '
)
else
RUNNING_CONTAINERS=$( $ENGINE ps --format '{{.Names}}' | tr '\n' ' ' )
fi
debug "Running containers: $RUNNING_CONTAINERS"
function exec_in_container() {
container=$1
local container=$1
shift;
if [ -n "$DOCKERCOMPOSE_FILE" ]
then
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE exec $container $@
if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
$COMPOSE_BIN -f "$DOCKERCOMPOSE_FILE" exec "$container" "$@"
return $?
fi
$ENGINE exec $container $@
$ENGINE exec "$container" "$@"
return $?
}
# Implement check inside a function to allow running it in parallel
# Parameters : [container] [output file]
function check_container() {
container="$1"
output_file="$2"
STATUS=""
for check_plugin in ${CHECK_PLUGINS[@]}
do
exec_in_container $container test -e $check_plugin > /dev/null 2>&1
if [ $? -ne 0 ]
then
local container="$1" output_file="$2" status="" check_plugin check_plugin_cmd=() status ex
for check_plugin in "${!CHECK_PLUGINS[@]}"; do
if ! exec_in_container "$container" test -e "$check_plugin" > /dev/null 2>&1; then
debug "$container - Plugin $check_plugin not found"
continue
fi
debug "$container - Plugin $check_plugin found, use it"
STATUS="$(exec_in_container $container ${CHECK_PLUGINS[${check_plugin}]} 2>&1)"
read -ra check_plugin_cmd <<< "${CHECK_PLUGINS[${check_plugin}]}"
status="$( exec_in_container "$container" "${check_plugin_cmd[@]}" 2>&1 )"
ex=$?
debug "$container - Plugin output: $STATUS"
debug "$container - Plugin output: $status"
debug "$container - Plugin exit code: $ex"
break
done
if [ -z "$STATUS" ]
then
if [[ -z "$status" ]]; then
debug "$container - No check plugin found"
STATUS="UNKNOWN - No check plugin available"
status="UNKNOWN - No check plugin available"
ex=3
fi
echo $STATUS > $output_file
echo -e "$status" > "$output_file"
return $ex
}
debug "Trigger check of all selected containers..."
for container in $RUNNING_CONTAINERS
do
if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]}
then
for container in $RUNNING_CONTAINERS; do
if ! is_empty "${ONLY_CONTAINERS[@]}" && ! in_array "$container" "${ONLY_CONTAINERS[@]}"; then
debug "$container - Ignored"
continue
fi
if in_array $container ${EXCLUDED_CONTAINERS[@]}
then
if in_array "$container" "${EXCLUDED_CONTAINERS[@]}"; then
debug "$container - Excluded"
continue
fi
if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ]
then
if [[ "$MAX_PARALLEL_CHECKS" -gt 0 ]] && \
[[ "$(jobs | wc -l)" -ge "$MAX_PARALLEL_CHECKS" ]]; then
debug "Max parallel checks count reached. Waiting some check ending"
wait -n
debug "Some check ended, continue"
fi
CHECKED_CONTAINERS+=( "$container" )
CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) )
check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! )
check_container "$container" "${CONTAINER_STATUS_FILE[$container]}" & CONTAINER_PID+=( ["$container"]=$! )
done
debug "Wait for each individual container check and handle their result..."
for container in ${!CONTAINER_PID[@]}
do
for container in "${!CONTAINER_PID[@]}"; do
pid=${CONTAINER_PID[$container]}
debug "$container - Waiting for PID ${pid}..."
wait $pid
wait "$pid"
ex=$?
debug "$container - Check return ${ex}"
STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} )
rm -f ${CONTAINER_STATUS_FILE[$container]}
if [ $ex -eq 0 ]
then
STATUS=$( cat "${CONTAINER_STATUS_FILE[$container]}" )
rm -f "${CONTAINER_STATUS_FILE[$container]}"
if [[ $ex -eq 0 ]]; then
UP_TO_DATE+=( ["$container"]=$STATUS )
else
ERRORS+=( ["$container"]=$STATUS )
[ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || \
if [[ $ex -ge 3 ]]; then
UNKNOWNS+=( "$container" )
else
UPGRADABLE_CONTAINERS+=( ["$container"]="$STATUS" )
fi
[ $EXIT_CODE -ge $ex ] && continue
[ $ex -gt 3 ] && ex=3
fi
[[ $EXIT_CODE -ge $ex ]] && continue
[[ $ex -gt 3 ]] && ex=3
EXIT_CODE=$ex
done
NOTFOUNDS=()
if ! is_empty $ONLY_CONTAINERS
then
for container in ${ONLY_CONTAINERS[@]}
do
if ! in_array $container ${CHECKED_CONTAINERS[@]}
then
if ! is_empty "${ONLY_CONTAINERS[@]}"; then
for container in "${ONLY_CONTAINERS[@]}"; do
if ! in_array "$container" "${CHECKED_CONTAINERS[@]}"; then
debug "$container - Container not found"
ERRORS+=( ["$container"]="Container not found" )
NOTFOUNDS+=( "$container" )
@ -455,7 +449,7 @@ debug "Containers with errors (${#ERRORS[@]}): $( implode ", " "${!ERRORS[@]}" )
debug "Not found containers (${#NOTFOUNDS[@]}): $( implode ", " "${NOTFOUNDS[@]}" )"
# Compute performance data
let CONTAINER_COUNTS=${#CHECKED_CONTAINERS[@]}+${#NOTFOUNDS[@]}
(( CONTAINER_COUNTS=${#CHECKED_CONTAINERS[@]}+${#NOTFOUNDS[@]} ))
PERF_DATA=(
"uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS"
"upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS"
@ -483,31 +477,30 @@ case $EXIT_CODE in
esac
# Trigger container build (if need, enabled and docker compose file is provided)
if [ $REBUILD -eq 1 ]
then
debug "Check if we have to trigger some rebuild (status file: $(rebuild_status_file))"
if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]
then
if [[ $REBUILD -eq 1 ]]; then
[[ -n "$REBUILD_STATUS_FILE" ]] || rebuild_status_file
debug "Check if we have to trigger some rebuild (status file: $REBUILD_STATUS_FILE)"
if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]; then
debug "No upgradable container to rebuild"
remove_rebuild_status
elif [ -z "$DOCKERCOMPOSE_FILE" ]
then
elif [[ -z "$DOCKERCOMPOSE_FILE" ]]; then
message
message "WARNING: No docker compose file provided, can't trigger rebuild of following" \
"container(s):"
message "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )"
message "- $( implode "\n- " "${UPGRADABLE_CONTAINERS[@]}" )"
else
message "Rebuilding containers:"
REBUILT_CONTAINERS=()
for container in ${!UPGRADABLE_CONTAINERS[@]}; do
for container in "${!UPGRADABLE_CONTAINERS[@]}"; do
need_rebuild=0
container_data=$( rebuild_status "$container" )
if [[ "$container_data" != "null" ]]; then
debug "$container: data=$container_data"
trigger_date=$( jq -r .trigger_date <<< $container_data )
start_date=$( jq -r .start_date <<< $container_data )
end_date=$( jq -r .end_date <<< $container_data )
log=$( jq -r .log <<< $container_data )
rebuild_status CONTAINER_DATA "$container"
# shellcheck disable=SC2153
if [[ "$CONTAINER_DATA" != "null" ]]; then
debug "$container: data='$CONTAINER_DATA'"
trigger_date=$( jq -r .trigger_date <<< "$CONTAINER_DATA" )
start_date=$( jq -r .start_date <<< "$CONTAINER_DATA" )
end_date=$( jq -r .end_date <<< "$CONTAINER_DATA" )
log=$( jq -r .log <<< "$CONTAINER_DATA" )
if [[ "$start_date" == "null" ]]; then
debug "$container: build triggered but not yet started"
message "- $container: rebuild triggered on $trigger_date and not yet started"
@ -517,18 +510,18 @@ then
message "- $container: rebuild triggered on $trigger_date and started on" \
"$start_date, but not yet finish (log: $log)"
else
duration=$( jq -r .duration <<< $container_data )
duration=$( jq -r .duration <<< "$CONTAINER_DATA" )
debug "$container: rebuilt in $duration on $start_date (finish on $end_date)"
prev_status=$( jq -r .status <<< $container_data )
prev_status=$( jq -r .status <<< "$CONTAINER_DATA" )
if [[ "$prev_status" == "${UPGRADABLE_CONTAINERS[$container]}" ]]; then
error=$( jq -r .error <<< $container_data )
error=$( jq -r .error <<< "$CONTAINER_DATA" )
if [[ "$error" != "null" ]]; then
message "- $container: $error (log: $log)"
else
message "- $container: already rebuilt in $duration (rebuild" \
"triggered on $trigger_date, started on $start_date and finish" \
"at $end_date, log: $log)"
REBUILT_CONTAINERS+=( $container )
REBUILT_CONTAINERS+=( "$container" )
fi
else
update_rebuild_status -d "$container"
@ -556,21 +549,19 @@ then
message "Some containers are ready to be recreated and restarted."
message "Run the following command to do it:"
message
message " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[@]}"
message " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[*]}"
fi
message
fi
fi
# Display details, starting by errors
for container in ${!ERRORS[@]}
do
message ${container} - ${ERRORS[${container}]}
for container in "${!ERRORS[@]}"; do
message "${container}" - "${ERRORS[${container}]}"
done
for container in ${!UP_TO_DATE[@]}
do
message ${container} - ${UP_TO_DATE[${container}]}
for container in "${!UP_TO_DATE[@]}"; do
message "${container}" - "${UP_TO_DATE[${container}]}"
done
exit $EXIT_CODE