From 525473a7fc4ab4f2fb1bd62371a92f01b70418f3 Mon Sep 17 00:00:00 2001 From: Benjamin Renard Date: Mon, 8 Jul 2024 02:15:21 +0200 Subject: [PATCH] Improve rebuild feature: handle check upgrade status changes, run rebuild via cron and add a deploy cron job --- README.md | 29 +- check_container_upgrade | 719 +++++++++++++++++++++++++--------------- 2 files changed, 479 insertions(+), 269 deletions(-) diff --git a/README.md b/README.md index 55de8d4..9d1c457 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,25 @@ service nagios-nrpe-server reload ``` Usage : check_container_upgrade [-d] [-E /path/to/engine] [container1,...] - -E [path] Force a specific engine (possible values: auto docker podman, default: auto) - -x [container] Exclude specified container (could be repeat) - -M [integer] Max number of container checks to run in parallel (default: 4, 0=no limit) - -f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the docker-compose.yml file - -b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker compose file if provided) - --rebuild-lock Specify rebuild lock file path (default: /var/tmp/check_container_upgrade.lock) - --rebuild-log Specify rebuild log file path (default: /var/tmp/check_container_upgrade.log) - -d Debug mode - -X Enable bash tracing (=set -x) - -h Show this message + -E [path] Force a specific engine (possible values: auto docker podman, + default: auto) + -x [container] Exclude specified container (could be repeat) + -M [integer] Max number of container checks to run in parallel + (default: 4, 0=no limit) + -f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the + docker-compose.yml file + -b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker + compose file if provided) + --rebuild-path Specify rebuild data directory path (default: /var/log/check_container_upgrade) + --rebuild-cron Start in rebuild cron mode: rebuild containers detected and mark to be + rebuilt on status file. + --deploy-cron Start in deploy cron mode: deploy containers known as rebuilt in status + file. + -d Debug mode + -l Log file + -C Console logging (even if log file is specify) + -X Enable bash tracing (=set -x) + -h Show this message ``` ## Copyright diff --git a/check_container_upgrade b/check_container_upgrade index ff602df..05100b1 100755 --- a/check_container_upgrade +++ b/check_container_upgrade @@ -1,6 +1,6 @@ #!/bin/bash # Monitoring plugin to check if running containers are upgradable -# +# # Author: Benjamin Renard # Date: Sun, 03 Mar 2024 16:40:19 +0100 # Source: https://gitea.zionetrix.net/bn8/check_container_upgrade @@ -10,64 +10,154 @@ ENGINE="auto" POSSIBLE_ENGINES=( "auto" "docker" "podman" ) DOCKERCOMPOSE_FILE="" DEBUG=0 +CONSOLE=0 +LOG_FILE=0 MAX_PARALLEL_CHECKS=4 ONLY_CONTAINERS=() EXCLUDED_CONTAINERS=( buildx_buildkit_default ) REBUILD=0 -REBUILD_LOCK_FILE="/var/tmp/$( basename $0 ).lock" -REBUILD_LOG_FILE="/var/tmp/$( basename $0 ).log" +REBUILD_DATA_DIR="/var/log/$(basename $0)" +REBUILD_CRON=0 +DEPLOY_CRON=0 declare -rA CHECK_PLUGINS=( - ["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l" - ["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk" + ["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l" + ["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk" ) +function now() { [[ -z "$1" ]] && date "+%F %H:%M:%S" || date -d "@$1" "+%F %H:%M:%S" ; } +function current_time() { date "+%s"; } + function debug() { - if [ $DEBUG -eq 1 ] - then - >&2 echo -e "[DEBUG] $@" - fi + [[ $DEBUG -eq 0 ]] && return + if [[ -n "$LOG_FILE" ]] && [[ $CONSOLE -eq 1 ]]; then + echo -e "$(now) - [DEBUG] $@" | tee -a "$LOG_FILE" 2>&1 + elif [[ -n "$LOG_FILE" ]]; then + echo -e "$(now) - [DEBUG] $@" >> "$LOG_FILE" + else + >&2 echo -e "[DEBUG] $@" + fi } +function log() { + [[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${@:2}" >> "$LOG_FILE" + [[ "$1" == "ERROR" ]] && >&2 echo -e "ERROR - ${@:2}" || echo -e "${@:2}" +} + +function message() { log INFO $@ ; } +function error() { log ERROR $@ ; } + function is_empty() { - [ $# -gt 0 ] && return 1 - return 0 + [ $# -gt 0 ] && return 1 + return 0 } function in_array() { - param=$1; - shift; - for elem in "$@"; - do - [[ "$param" = "$elem" ]] && return 0; - done; - return 1 + param=$1; + shift; + for elem in "$@"; + do + [[ "$param" = "$elem" ]] && return 0; + done; + return 1 } function implode() { - local d=${1-} f=${2-} - if shift 2; then - printf %s "$f" "${@/#/$d}" - fi + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi +} + +function format_duration { + local T=$1 + local D=$((T/60/60/24)) + local H=$((T/60/60%24)) + local M=$((T/60%60)) + local S=$((T%60)) + (( $D > 0 )) && printf '%d days and ' $D + printf '%02d:%02d:%02d' $H $M $S +} + +REBUILD_STATUS_FILE="" +function rebuild_status_file() { + [[ ! -e "$REBUILD_DATA_DIR" ]] && mkdir -p "$REBUILD_DATA_DIR" + [[ -z "$REBUILD_STATUS_FILE" ]] && \ + REBUILD_STATUS_FILE="$REBUILD_DATA_DIR/status.json" + [[ -e "$REBUILD_STATUS_FILE" ]] || echo '{}' > "$REBUILD_STATUS_FILE" + echo "$REBUILD_STATUS_FILE" +} + +function rebuild_status() { + if [[ -z "$1" ]]; then + cat "$(rebuild_status_file)" + else + rebuild_status | jq -r --arg container "$1" '.[$container]' + fi +} + +function update_rebuild_status() { + if [[ "$1" == "-d" ]]; then + local data=$( + rebuild_status | jq \ + --arg container "$2" \ + 'del(.[$container])' + ) + else + local args=( --arg container "$1" ) + local expr=( ) + for arg in "${@:2}"; do + local name=$( echo "$arg" | cut -d'=' -f1 ) + local value=$( echo "$arg" | sed 's/[^=]\+=//' ) + args+=( --arg "$name" "$value" ) + expr+=( ".[\$container].$name=\$$name" ) + done + local data=$( + rebuild_status | jq \ + "${args[@]}" "$( implode ' | ' "${expr[@]}" )" + ) + fi + cat <<< $data > $(rebuild_status_file) +} + +function remove_rebuild_status() { + rebuild_status_file > /dev/null + [[ -e "$REBUILD_STATUS_FILE" ]] || return 0 + debug "Remove previous rebuild status file ($REBUILD_STATUS_FILE) and log files it contains" + for log in $( jq -r '.[] | .log' $REBUILD_STATUS_FILE ); do + debug " remove old container log file $log" + rm -f $log + done + debug " remove status file" + rm -f "$REBUILD_STATUS_FILE" } function usage() { - error="$1" - [ -n "$error" ] && echo "$error" - cat << EOF + error="$1" + [ -n "$error" ] && message "$error" + cat << EOF Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...] - -E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]}, default: $ENGINE) - -x [container] Exclude specified container (could be repeat) - -M [integer] Max number of container checks to run in parallel (default: $MAX_PARALLEL_CHECKS, 0=no limit) - -f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the docker-compose.yml file - -b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker compose file if provided) - --rebuild-lock Specify rebuild lock file path (default: ${REBUILD_LOCK_FILE}) - --rebuild-log Specify rebuild log file path (default: ${REBUILD_LOG_FILE}) - -d Debug mode - -X Enable bash tracing (=set -x) - -h Show this message + -E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]}, + default: $ENGINE) + -x [container] Exclude specified container (could be repeat) + -M [integer] Max number of container checks to run in parallel + (default: $MAX_PARALLEL_CHECKS, 0=no limit) + -f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the + docker-compose.yml file + -b|--build|--rebuild Trigger container build if upgrade detected (only possible if a docker + compose file if provided) + --rebuild-path Specify rebuild data directory path (default: ${REBUILD_DATA_DIR}) + --rebuild-cron Start in rebuild cron mode: rebuild containers detected and mark to be + rebuilt on status file. + --deploy-cron Start in deploy cron mode: deploy containers known as rebuilt in status + file. + -d Debug mode + -l Log file + -C Console logging (even if log file is specify) + -X Enable bash tracing (=set -x) + -h Show this message EOF - [ -n "$error" ] && exit 1 - exit 0 + [ -n "$error" ] && exit 1 + exit 0 } idx=1 @@ -75,82 +165,165 @@ while [ $idx -le $# ] do OPT=${!idx} case $OPT in - -d) - DEBUG=1 - ;; - -h) - usage - ;; - -E) - ((idx++)) - ENGINE=${!idx} - if [ ! -x "$ENGINE" ] - then - in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE" - fi - ;; - -f) - ((idx++)) - DOCKERCOMPOSE_FILE=${!idx} - ;; - -b|--build|--rebuild) - REBUILD=1 - ;; - --rebuild-lock) - ((idx++)) - REBUILD_LOCK_FILE="${!idx}" - ;; - --rebuild-log) - ((idx++)) - REBUILD_LOG_FILE="${!idx}" - ;; - -x) - ((idx++)) - EXCLUDED_CONTAINERS+=( ${!idx} ) - ;; - -M) - ((idx++)) - MAX_PARALLEL_CHECKS=${!idx} - ;; - -X) - set -x - ;; - *) - ONLY_CONTAINERS+=( $OPT ) - ;; + -d) + DEBUG=1 + ;; + -C) + CONSOLE=1 + ;; + -l) + ((idx++)) + LOG_FILE=${!idx} + ;; + -h) + usage + ;; + -E) + ((idx++)) + ENGINE=${!idx} + if [ ! -x "$ENGINE" ] + then + in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE" + fi + ;; + -f) + ((idx++)) + DOCKERCOMPOSE_FILE=${!idx} + ;; + -b|--build|--rebuild) + REBUILD=1 + ;; + --rebuild-path) + ((idx++)) + REBUILD_DATA_DIR="${!idx}" + ;; + -x) + ((idx++)) + EXCLUDED_CONTAINERS+=( ${!idx} ) + ;; + -M) + ((idx++)) + MAX_PARALLEL_CHECKS=${!idx} + ;; + -X) + set -x + ;; + --rebuild-cron) + REBUILD_CRON=1 + ;; + --deploy-cron) + DEPLOY_CRON=1 + ;; + *) + ONLY_CONTAINERS+=( $OPT ) + ;; esac ((idx++)) done +debug "Start with parameters: $@" + ! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}" if [ "$ENGINE" == "auto" ] then - debug "Auto-detect engine..." - for engine in ${POSSIBLE_ENGINES[@]} - do - [ "$engine" == "auto" ] && continue - which "$engine" > /dev/null 2>&1 - if [ $? -ne 0 ] - then - debug "$engine not found" - continue - fi - ENGINE="$engine" - break - done - if [ -z "$ENGINE" ] - then - echo "UNKNOWN - Fail to auto-detect engine" - exit 3 - fi - debug "Auto-detected engine: $ENGINE" + debug "Auto-detect engine..." + for engine in ${POSSIBLE_ENGINES[@]} + do + [ "$engine" == "auto" ] && continue + which "$engine" > /dev/null 2>&1 + if [ $? -ne 0 ] + then + debug "$engine not found" + continue + fi + ENGINE="$engine" + break + done + if [ -z "$ENGINE" ] + then + message "UNKNOWN - Fail to auto-detect engine" + exit 3 + fi + debug "Auto-detected engine: $ENGINE" fi -if [ -n "$DOCKERCOMPOSE_FILE" -a ! -e "$DOCKERCOMPOSE_FILE" ] -then - echo "UNKNOWN - Docker compose file not found ($DOCKERCOMPOSE_FILE)" - exit 3 +if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then + if [[ ! -e "$DOCKERCOMPOSE_FILE" ]]; then + message "UNKNOWN - Docker compose file not found ($DOCKERCOMPOSE_FILE)" + exit 3 + fi + which "${ENGINE}-compose" > /dev/null 2>&1 && \ + COMPOSE_BIN="${ENGINE}-compose" || \ + COMPOSE_BIN="$ENGINE compose" + debug "Docker compose bin: $COMPOSE_BIN" +fi + +if [[ $REBUILD_CRON -eq 1 ]]; then + to_rebuild=( + $( + rebuild_status | jq -r -c \ + 'to_entries[] | select((.value.start_date|not) and (.value.error|not)) | .key' + ) + ) + if [[ ${#to_rebuild[@]} -eq 0 ]]; then + debug "No container need to be rebuild" + exit 0 + fi + message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[@]}" + error=0 + for container in ${to_rebuild[@]}; do + log="$REBUILD_DATA_DIR/$container.log" + message " $container: start building image (log=$log)" + start_time=$(current_time) + update_rebuild_status "$container" "start_date=$(now $start_time)" "log=$log" + $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache $container >> $log 2>&1 + result=$? + end_time=$(current_time) + (( duration=end_time-start_time )) + duration=$(format_duration $duration) + container_info=( "end_date=$(now $end_time)" "duration=$duration" ) + if [[ $result -eq 0 ]]; then + message " $container: rebuilt in $duration" + else + error " $container: fail to rebuild image" + container_info+=( "error=fail to rebuild image" ) + fi + update_rebuild_status "$container" "${container_info[@]}" + done + message "No more container to rebuild, stop" + exit 0 +fi + +if [[ $DEPLOY_CRON -eq 1 ]]; then + to_deploy=( + $( + rebuild_status | jq -r -c \ + 'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key' + ) + ) + if [[ ${#to_deploy[@]} -eq 0 ]]; then + debug "No container need to be deploy" + exit 0 + fi + message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[@]}" + error=0 + for container in ${to_deploy[@]}; do + message " $container: deploying..." + log="$REBUILD_DATA_DIR/$container.log" + $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps $container > $log 2>&1 + if [[ $? -eq 0 ]]; then + message " $container: done" + update_rebuild_status -d "$container" + rm -f $log + else + error " $container: fail to deploy new container image" + update_rebuild_status "$container" "error=fail to deploy new container image" + error=1 + fi + done + message "done" + exit $error fi EXIT_CODE=0 @@ -159,219 +332,247 @@ declare -A CONTAINER_PID declare -A UP_TO_DATE declare -A ERRORS declare -A UNKNOWNS -UPGRADABLE_CONTAINERS=( ) +declare -A UPGRADABLE_CONTAINERS CHECKED_CONTAINERS=( ) debug "List running containers..." -if [ -n "$DOCKERCOMPOSE_FILE" ] -then - which "${ENGINE}-compose" > /dev/null 2>&1 \ - && COMPOSE_BIN="${ENGINE}-compose" \ - || COMPOSE_BIN="$ENGINE compose" - RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ') -else - RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ') -fi +[[ -n "$DOCKERCOMPOSE_FILE" ]] && \ + RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ') || + RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ') debug "Running containers: $RUNNING_CONTAINERS" function exec_in_container() { - container=$1 - shift; - if [ -n "$DOCKERCOMPOSE_FILE" ] - then - $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE exec $container $@ - return $? - fi - $ENGINE exec $container $@ - return $? + container=$1 + shift; + if [ -n "$DOCKERCOMPOSE_FILE" ] + then + $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE exec $container $@ + return $? + fi + $ENGINE exec $container $@ + return $? } # Implement check inside a function to allow running it in parallel # Parameters : [container] [output file] function check_container() { - container="$1" - output_file="$2" - STATUS="" - for check_plugin in ${CHECK_PLUGINS[@]} - do - exec_in_container $container test -e $check_plugin > /dev/null 2>&1 - if [ $? -ne 0 ] - then - debug "$container - Plugin $check_plugin not found" - continue - fi - debug "$container - Plugin $check_plugin found, use it" - STATUS="$(exec_in_container $container ${CHECK_PLUGINS[${check_plugin}]} 2>&1)" - ex=$? - debug "$container - Plugin output: $STATUS" - debug "$container - Plugin exit code: $ex" - break - done - if [ -z "$STATUS" ] - then - debug "$container - No check plugin found" - STATUS="UNKNOWN - No check plugin available" - ex=3 - fi - echo $STATUS > $output_file - return $ex + container="$1" + output_file="$2" + STATUS="" + for check_plugin in ${CHECK_PLUGINS[@]} + do + exec_in_container $container test -e $check_plugin > /dev/null 2>&1 + if [ $? -ne 0 ] + then + debug "$container - Plugin $check_plugin not found" + continue + fi + debug "$container - Plugin $check_plugin found, use it" + STATUS="$(exec_in_container $container ${CHECK_PLUGINS[${check_plugin}]} 2>&1)" + ex=$? + debug "$container - Plugin output: $STATUS" + debug "$container - Plugin exit code: $ex" + break + done + if [ -z "$STATUS" ] + then + debug "$container - No check plugin found" + STATUS="UNKNOWN - No check plugin available" + ex=3 + fi + echo $STATUS > $output_file + return $ex } debug "Trigger check of all selected containers..." for container in $RUNNING_CONTAINERS do - if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]} - then - debug "$container - Ignored" - continue - fi - if in_array $container ${EXCLUDED_CONTAINERS[@]} - then - debug "$container - Excluded" - continue - fi - if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ] - then - debug "Max parallel checks count reached. Waiting some check ending" - wait -n - debug "Some check ended, continue" - fi - CHECKED_CONTAINERS+=( "$container" ) - CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) ) - check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! ) + if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]} + then + debug "$container - Ignored" + continue + fi + if in_array $container ${EXCLUDED_CONTAINERS[@]} + then + debug "$container - Excluded" + continue + fi + if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ] + then + debug "Max parallel checks count reached. Waiting some check ending" + wait -n + debug "Some check ended, continue" + fi + CHECKED_CONTAINERS+=( "$container" ) + CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) ) + check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! ) done debug "Wait for each individual container check and handle their result..." for container in ${!CONTAINER_PID[@]} do - pid=${CONTAINER_PID[$container]} - debug "$container - Waiting for PID ${pid}..." - wait $pid - ex=$? - debug "$container - Check return ${ex}" - STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} ) - rm -f ${CONTAINER_STATUS_FILE[$container]} - if [ $ex -eq 0 ] - then - UP_TO_DATE+=( ["$container"]=$STATUS ) - else - ERRORS+=( ["$container"]=$STATUS ) - [ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || UPGRADABLE_CONTAINERS+=( "$container" ) - fi - [ $EXIT_CODE -ge $ex ] && continue - [ $ex -gt 3 ] && ex=3 - EXIT_CODE=$ex + pid=${CONTAINER_PID[$container]} + debug "$container - Waiting for PID ${pid}..." + wait $pid + ex=$? + debug "$container - Check return ${ex}" + STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} ) + rm -f ${CONTAINER_STATUS_FILE[$container]} + if [ $ex -eq 0 ] + then + UP_TO_DATE+=( ["$container"]=$STATUS ) + else + ERRORS+=( ["$container"]=$STATUS ) + [ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || \ + UPGRADABLE_CONTAINERS+=( ["$container"]="$STATUS" ) + fi + [ $EXIT_CODE -ge $ex ] && continue + [ $ex -gt 3 ] && ex=3 + EXIT_CODE=$ex done NOTFOUNDS=() if ! is_empty $ONLY_CONTAINERS then - for container in ${ONLY_CONTAINERS[@]} - do - if ! in_array $container ${CHECKED_CONTAINERS[@]} - then - debug "$container - Container not found" - ERRORS+=( ["$container"]="Container not found" ) - NOTFOUNDS+=( "$container" ) - EXIT_CODE=3 - fi - done + for container in ${ONLY_CONTAINERS[@]} + do + if ! in_array $container ${CHECKED_CONTAINERS[@]} + then + debug "$container - Container not found" + ERRORS+=( ["$container"]="Container not found" ) + NOTFOUNDS+=( "$container" ) + EXIT_CODE=3 + fi + done fi debug "Final exit code: $EXIT_CODE" debug "Check containers (${#CHECKED_CONTAINERS[@]}): $( implode ", " "${CHECKED_CONTAINERS[@]}" )" debug "Up-to-date containers (${#UP_TO_DATE[@]}): $( implode ", " "${!UP_TO_DATE[@]}" )" -debug "Upgradable containers (${#UPGRADABLE_CONTAINERS[@]}): $( implode ", " "${UPGRADABLE_CONTAINERS[@]}" )" +debug "Upgradable containers (${#UPGRADABLE_CONTAINERS[@]}): $( implode ", " "${!UPGRADABLE_CONTAINERS[@]}" )" debug "Containers with errors (${#ERRORS[@]}): $( implode ", " "${!ERRORS[@]}" )" debug "Not found containers (${#NOTFOUNDS[@]}): $( implode ", " "${NOTFOUNDS[@]}" )" # Compute performance data let CONTAINER_COUNTS=${#CHECKED_CONTAINERS[@]}+${#NOTFOUNDS[@]} PERF_DATA=( - "uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS" - "upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS" - "containers_with_errors=${#ERRORS[@]};1;;0;$CONTAINER_COUNTS" - "unknown_state_containers=${#UNKNOWNS[@]};;;0;$CONTAINER_COUNTS" + "uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS" + "upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS" + "containers_with_errors=${#ERRORS[@]};1;;0;$CONTAINER_COUNTS" + "unknown_state_containers=${#UNKNOWNS[@]};;;0;$CONTAINER_COUNTS" ) +# Compute performance data as string +PERF_DATA_TXT="$( implode " " "${PERF_DATA[@]}" )" + # Display check result message case $EXIT_CODE in - 0) - echo -n "OK - All ${#UP_TO_DATE[@]} container(s) are up-to-date" - ;; - 1) - echo -n "WARNING - ${#ERRORS[@]} container(s) need to be updated" - ;; - 2) - echo -n "CRITICAL - ${#ERRORS[@]} container(s) need to be updated" - ;; - *) - echo -n "UNKNOWN - fail to retrieve status of ${#UNKNOWNS[@]} container(s)" - ;; + 0) + message "OK - All ${#UP_TO_DATE[@]} container(s) are up-to-date |$PERF_DATA_TXT" + ;; + 1) + message "WARNING - ${#ERRORS[@]} container(s) need to be updated |$PERF_DATA_TXT" + ;; + 2) + message "CRITICAL - ${#ERRORS[@]} container(s) need to be updated |$PERF_DATA_TXT" + ;; + *) + message "UNKNOWN - fail to retrieve status of ${#UNKNOWNS[@]} container(s) |$PERF_DATA_TXT" + ;; esac -# Add performance data -echo " |$( implode " " "${PERF_DATA[@]}" )" - # Trigger container build (if need, enabled and docker compose file is provided) if [ $REBUILD -eq 1 ] then - if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ] - then - debug "No upgradable container to rebuild" - if [ -e "$REBUILD_LOCK_FILE" ] - then - debug "Remove previous rebuild lock file ($REBUILD_LOCK_FILE)" - rm -f "$REBUILD_LOCK_FILE" - fi - elif [ -z "$DOCKERCOMPOSE_FILE" ] - then - echo - echo "WARNING: No docker compose file provided, can't trigger rebuild of following container(s):" - echo "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )" - elif [ -e "$REBUILD_LOCK_FILE" ] - then - REBUILD_PID=$(cat $REBUILD_LOCK_FILE) - echo - if [ -d "/proc/$REBUILD_PID" ] - then - echo "Rebuild already triggered on $( stat --format %y "$REBUILD_LOCK_FILE" ) (PID: $REBUILD_PID)" - echo "You could follow it in its log file: ${REBUILD_LOG_FILE}" - else - echo "Container images already rebuilt (see ${REBUILD_LOG_FILE} for details)." - echo "You could recreate and restart them using the following command:" - echo - echo " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${UPGRADABLE_CONTAINERS[@]}" - echo - echo "Note:" - echo " If new updates was published since rebuild was triggered, remove the" - echo " '$REBUILD_LOCK_FILE' file and recheck it to trigger a new build." - fi - echo - else - echo - echo "Trigger rebuild of following container(s): ${UPGRADABLE_CONTAINERS[@]}" - echo "You could follow the rebuild process in its log file: ${REBUILD_LOG_FILE}" - echo "You will able to recreate and restart them using the following command:" - echo - echo " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${UPGRADABLE_CONTAINERS[@]}" - echo + debug "Check if we have to trigger some rebuild (status file: $(rebuild_status_file))" + if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ] + then + debug "No upgradable container to rebuild" + remove_rebuild_status + elif [ -z "$DOCKERCOMPOSE_FILE" ] + then + message + message "WARNING: No docker compose file provided, can't trigger rebuild of following" \ + "container(s):" + message "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )" + else + message "Rebuilding containers:" + REBUILT_CONTAINERS=() + for container in ${!UPGRADABLE_CONTAINERS[@]}; do + need_rebuild=0 + container_data=$( rebuild_status "$container" ) + if [[ "$container_data" != "null" ]]; then + debug "$container: data=$container_data" + trigger_date=$( jq -r .trigger_date <<< $container_data ) + start_date=$( jq -r .start_date <<< $container_data ) + end_date=$( jq -r .end_date <<< $container_data ) + log=$( jq -r .log <<< $container_data ) + if [[ "$start_date" == "null" ]]; then + debug "$container: build triggered but not yet started" + message "- $container: rebuild triggered on $trigger_date and not yet started" + elif [[ "$end_date" == "null" ]]; then + debug "$container: rebuild triggered on $trigger_date and started on" \ + "$start_date, but not yet finish" + message "- $container: rebuild triggered on $trigger_date and started on" \ + "$start_date, but not yet finish (log: $log)" + else + duration=$( jq -r .duration <<< $container_data ) + debug "$container: rebuilt in $duration on $start_date (finish on $end_date)" + prev_status=$( jq -r .status <<< $container_data ) + if [[ "$prev_status" == "${UPGRADABLE_CONTAINERS[$container]}" ]]; then + error=$( jq -r .error <<< $container_data ) + if [[ "$error" != "null" ]]; then + message "- $container: $error (log: $log)" + else + message "- $container: already rebuilt in $duration (rebuild" \ + "triggered on $trigger_date, started on $start_date and finish" \ + "at $end_date,log: $log)" + REBUILT_CONTAINERS+=( $container ) + fi + else + update_rebuild_status -d "$container" + message "- $container: upgrade status change since last rebuild, rebuild" \ + "it again" + need_rebuild=1 + fi + fi + else + debug "$container: not found in status file" + need_rebuild=1 + fi + if [[ $need_rebuild -eq 1 ]]; then + update_rebuild_status "$container" \ + "trigger_date=$(now)" "status=${UPGRADABLE_CONTAINERS[$container]}" + message "- $container: rebuild triggered" + else + debug "$container: rebuild not need" + fi + done - $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache ${UPGRADABLE_CONTAINERS[@]} > $REBUILD_LOG_FILE 2>&1 & - echo $! > "$REBUILD_LOCK_FILE" - fi + # Handle rebuilt containers + if [[ ${#REBUILT_CONTAINERS[@]} -gt 0 ]]; then + message + message "Some containers are ready to be recreated and restarted." + message "Run the following command to do it:" + message + message " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[@]}" + fi + message + fi fi # Display details, starting by errors for container in ${!ERRORS[@]} do - echo ${container} - ${ERRORS[${container}]} + message ${container} - ${ERRORS[${container}]} done for container in ${!UP_TO_DATE[@]} do - echo ${container} - ${UP_TO_DATE[${container}]} + message ${container} - ${UP_TO_DATE[${container}]} done exit $EXIT_CODE + +# vim: shiftwidth=4 tabstop=4 expandtab