Improve rebuild feature: handle check upgrade status changes, run rebuild via cron and add a deploy cron job

check_container_upgrade

@@ -10,24 +10,42 @@ ENGINE="auto"
 POSSIBLE_ENGINES=( "auto" "docker" "podman" )
 DOCKERCOMPOSE_FILE=""
 DEBUG=0
+CONSOLE=0
+LOG_FILE=0
 MAX_PARALLEL_CHECKS=4
 ONLY_CONTAINERS=()
 EXCLUDED_CONTAINERS=( buildx_buildkit_default )
 REBUILD=0
-REBUILD_LOCK_FILE="/var/tmp/$( basename $0 ).lock"
+REBUILD_DATA_DIR="/var/log/$(basename $0)"
-REBUILD_LOG_FILE="/var/tmp/$( basename $0 ).log"
+REBUILD_CRON=0
+DEPLOY_CRON=0
 declare -rA CHECK_PLUGINS=(
     ["/usr/lib/nagios/plugins/check_apt"]="/usr/lib/nagios/plugins/check_apt -u -U -t 60 -l"
     ["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk"
 )
 
+function now() { [[ -z "$1" ]] && date "+%F %H:%M:%S" || date -d "@$1" "+%F %H:%M:%S" ; }
+function current_time() { date "+%s"; }
+
 function debug() {
-	if [ $DEBUG -eq 1 ]
+    [[ $DEBUG -eq 0 ]] && return
-	then
+    if [[ -n "$LOG_FILE" ]] && [[ $CONSOLE -eq 1 ]]; then
+        echo -e "$(now) - [DEBUG] $@" | tee -a "$LOG_FILE" 2>&1
+    elif [[ -n "$LOG_FILE" ]]; then
+        echo -e "$(now) - [DEBUG] $@" >> "$LOG_FILE"
+    else
         >&2 echo -e "[DEBUG] $@"
     fi
 }
 
+function log() {
+    [[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${@:2}" >> "$LOG_FILE"
+    [[ "$1" == "ERROR" ]] && >&2 echo -e "ERROR - ${@:2}" || echo -e "${@:2}"
+}
+
+function message() { log INFO $@ ; }
+function error() { log ERROR $@ ; }
+
 function is_empty() {
     [ $# -gt 0 ] && return 1
     return 0
@@ -50,19 +68,91 @@ function implode() {
     fi
 }
 
+function format_duration {
+    local T=$1
+    local D=$((T/60/60/24))
+    local H=$((T/60/60%24))
+    local M=$((T/60%60))
+    local S=$((T%60))
+    (( $D > 0 )) && printf '%d days and ' $D
+    printf '%02d:%02d:%02d' $H $M $S
+}
+
+REBUILD_STATUS_FILE=""
+function rebuild_status_file() {
+    [[ ! -e "$REBUILD_DATA_DIR" ]] && mkdir -p "$REBUILD_DATA_DIR"
+    [[ -z "$REBUILD_STATUS_FILE" ]] && \
+        REBUILD_STATUS_FILE="$REBUILD_DATA_DIR/status.json"
+    [[ -e "$REBUILD_STATUS_FILE" ]] || echo '{}' > "$REBUILD_STATUS_FILE"
+    echo "$REBUILD_STATUS_FILE"
+}
+
+function rebuild_status() {
+    if [[ -z "$1" ]]; then
+        cat "$(rebuild_status_file)"
+    else
+        rebuild_status | jq -r --arg container "$1" '.[$container]'
+    fi
+}
+
+function update_rebuild_status() {
+    if [[ "$1" == "-d" ]]; then
+        local data=$(
+            rebuild_status | jq \
+                --arg container "$2" \
+                'del(.[$container])'
+        )
+    else
+        local args=( --arg container "$1" )
+        local expr=( )
+        for arg in "${@:2}"; do
+            local name=$( echo "$arg" | cut -d'=' -f1 )
+            local value=$( echo "$arg" | sed 's/[^=]\+=//' )
+            args+=( --arg "$name" "$value" )
+            expr+=( ".[\$container].$name=\$$name" )
+        done
+        local data=$(
+            rebuild_status | jq \
+            "${args[@]}" "$( implode ' | ' "${expr[@]}" )"
+        )
+    fi
+    cat <<< $data > $(rebuild_status_file)
+}
+
+function remove_rebuild_status() {
+    rebuild_status_file > /dev/null
+    [[ -e "$REBUILD_STATUS_FILE" ]] || return 0
+    debug "Remove previous rebuild status file ($REBUILD_STATUS_FILE) and log files it contains"
+    for log in $( jq -r '.[] | .log' $REBUILD_STATUS_FILE ); do
+        debug "  remove old container log file $log"
+        rm -f $log
+    done
+    debug "  remove status file"
+    rm -f "$REBUILD_STATUS_FILE"
+}
+
 function usage() {
     error="$1"
-	[ -n "$error" ] && echo "$error"
+    [ -n "$error" ] && message "$error"
     cat << EOF
 Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
-	-E [path]               Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]}, default: $ENGINE)
+    -E [path]               Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]},
+                            default: $ENGINE)
     -x [container]          Exclude specified container (could be repeat)
-	-M [integer]		Max number of container checks to run in parallel (default: $MAX_PARALLEL_CHECKS, 0=no limit)
+    -M [integer]            Max number of container checks to run in parallel
-	-f [docker-compose.yml]	To check upgrade on docker compose project, specified the path of the docker-compose.yml file
+                            (default: $MAX_PARALLEL_CHECKS, 0=no limit)
-	-b|--build|--rebuild	Trigger container build if upgrade detected (only possible if a docker compose file if provided)
+    -f [docker-compose.yml] To check upgrade on docker compose project, specified the path of the
-	--rebuild-lock		Specify rebuild lock file path (default: ${REBUILD_LOCK_FILE})
+                            docker-compose.yml file
-	--rebuild-log		Specify rebuild log file path (default: ${REBUILD_LOG_FILE})
+    -b|--build|--rebuild    Trigger container build if upgrade detected (only possible if a docker
+                            compose file if provided)
+    --rebuild-path          Specify rebuild data directory path (default: ${REBUILD_DATA_DIR})
+    --rebuild-cron          Start in rebuild cron mode: rebuild containers detected and mark to be
+                            rebuilt on status file.
+    --deploy-cron           Start in deploy cron mode: deploy containers known as rebuilt in status
+                            file.
     -d                      Debug mode
+    -l                      Log file
+    -C                      Console logging (even if log file is specify)
     -X                      Enable bash tracing (=set -x)
     -h                      Show this message
 EOF
@@ -78,6 +168,13 @@ do
         -d)
             DEBUG=1
         ;;
+        -C)
+            CONSOLE=1
+        ;;
+        -l)
+            ((idx++))
+            LOG_FILE=${!idx}
+        ;;
         -h)
             usage
         ;;
@@ -96,13 +193,9 @@ do
         -b|--build|--rebuild)
             REBUILD=1
         ;;
-		--rebuild-lock)
+        --rebuild-path)
             ((idx++))
-			REBUILD_LOCK_FILE="${!idx}"
+            REBUILD_DATA_DIR="${!idx}"
-		;;
-		--rebuild-log)
-			((idx++))
-			REBUILD_LOG_FILE="${!idx}"
         ;;
         -x)
             ((idx++))
@@ -115,6 +208,12 @@ do
         -X)
             set -x
         ;;
+        --rebuild-cron)
+            REBUILD_CRON=1
+        ;;
+        --deploy-cron)
+            DEPLOY_CRON=1
+        ;;
         *)
             ONLY_CONTAINERS+=( $OPT )
         ;;
@@ -122,6 +221,8 @@ do
         ((idx++))
 done
 
+debug "Start with parameters: $@"
+
 ! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}"
 
 if [ "$ENGINE" == "auto" ]
@@ -141,17 +242,89 @@ then
     done
     if [ -z "$ENGINE" ]
     then
-		echo "UNKNOWN - Fail to auto-detect engine"
+        message "UNKNOWN - Fail to auto-detect engine"
         exit 3
     fi
     debug "Auto-detected engine: $ENGINE"
 fi
 
-if [ -n "$DOCKERCOMPOSE_FILE" -a ! -e "$DOCKERCOMPOSE_FILE" ]
+if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
-then
+    if [[ ! -e "$DOCKERCOMPOSE_FILE" ]]; then
-	echo "UNKNOWN - Docker compose file not found ($DOCKERCOMPOSE_FILE)"
+        message "UNKNOWN - Docker compose file not found ($DOCKERCOMPOSE_FILE)"
         exit 3
     fi
+    which "${ENGINE}-compose" > /dev/null 2>&1 && \
+        COMPOSE_BIN="${ENGINE}-compose" || \
+        COMPOSE_BIN="$ENGINE compose"
+    debug "Docker compose bin: $COMPOSE_BIN"
+fi
+
+if [[ $REBUILD_CRON -eq 1 ]]; then
+    to_rebuild=(
+        $(
+            rebuild_status | jq -r -c \
+                'to_entries[] | select((.value.start_date|not) and (.value.error|not)) | .key'
+        )
+    )
+    if [[ ${#to_rebuild[@]} -eq 0 ]]; then
+        debug "No container need to be rebuild"
+        exit 0
+    fi
+    message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[@]}"
+    error=0
+    for container in ${to_rebuild[@]}; do
+        log="$REBUILD_DATA_DIR/$container.log"
+        message "  $container: start building image (log=$log)"
+        start_time=$(current_time)
+        update_rebuild_status "$container" "start_date=$(now $start_time)" "log=$log"
+        $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache $container >> $log 2>&1
+        result=$?
+        end_time=$(current_time)
+        (( duration=end_time-start_time ))
+        duration=$(format_duration $duration)
+        container_info=( "end_date=$(now $end_time)" "duration=$duration" )
+        if [[ $result -eq 0 ]]; then
+            message "  $container: rebuilt in $duration"
+        else
+            error "  $container: fail to rebuild image"
+            container_info+=( "error=fail to rebuild image" )
+        fi
+        update_rebuild_status "$container" "${container_info[@]}"
+    done
+    message "No more container to rebuild, stop"
+    exit 0
+fi
+
+if [[ $DEPLOY_CRON -eq 1 ]]; then
+    to_deploy=(
+        $(
+            rebuild_status | jq -r -c \
+                'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key'
+        )
+    )
+    if [[ ${#to_deploy[@]} -eq 0 ]]; then
+        debug "No container need to be deploy"
+        exit 0
+    fi
+    message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[@]}"
+    error=0
+    for container in ${to_deploy[@]}; do
+        message "  $container: deploying..."
+        log="$REBUILD_DATA_DIR/$container.log"
+        $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps $container > $log 2>&1
+        if [[ $? -eq 0 ]]; then
+            message "  $container: done"
+            update_rebuild_status -d "$container"
+			rm -f $log
+        else
+            error "  $container: fail to deploy new container image"
+            update_rebuild_status "$container" "error=fail to deploy new container image"
+            error=1
+        fi
+    done
+    message "done"
+    exit $error
+fi
 
 EXIT_CODE=0
 declare -A CONTAINER_STATUS_FILE
@@ -159,19 +332,13 @@ declare -A CONTAINER_PID
 declare -A UP_TO_DATE
 declare -A ERRORS
 declare -A UNKNOWNS
-UPGRADABLE_CONTAINERS=( )
+declare -A UPGRADABLE_CONTAINERS
 CHECKED_CONTAINERS=( )
 
 debug "List running containers..."
-if [ -n "$DOCKERCOMPOSE_FILE" ]
+[[ -n "$DOCKERCOMPOSE_FILE" ]] && \
-then
+    RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ') ||
-	which "${ENGINE}-compose" > /dev/null 2>&1 \
-		&& COMPOSE_BIN="${ENGINE}-compose" \
-		|| COMPOSE_BIN="$ENGINE compose"
-	RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ')
-else
     RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ')
-fi
 debug "Running containers: $RUNNING_CONTAINERS"
 
 function exec_in_container() {
@@ -256,7 +423,8 @@ do
         UP_TO_DATE+=( ["$container"]=$STATUS )
     else
         ERRORS+=( ["$container"]=$STATUS )
-		[ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || UPGRADABLE_CONTAINERS+=( "$container" )
+        [ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || \
+            UPGRADABLE_CONTAINERS+=( ["$container"]="$STATUS" )
     fi
     [ $EXIT_CODE -ge $ex ] && continue
     [ $ex -gt 3 ] && ex=3
@@ -282,7 +450,7 @@ debug "Final exit code: $EXIT_CODE"
 
 debug "Check containers (${#CHECKED_CONTAINERS[@]}): $( implode ", " "${CHECKED_CONTAINERS[@]}" )"
 debug "Up-to-date containers (${#UP_TO_DATE[@]}): $( implode ", " "${!UP_TO_DATE[@]}" )"
-debug "Upgradable containers (${#UPGRADABLE_CONTAINERS[@]}): $( implode ", " "${UPGRADABLE_CONTAINERS[@]}" )"
+debug "Upgradable containers (${#UPGRADABLE_CONTAINERS[@]}): $( implode ", " "${!UPGRADABLE_CONTAINERS[@]}" )"
 debug "Containers with errors (${#ERRORS[@]}): $( implode ", " "${!ERRORS[@]}" )"
 debug "Not found containers (${#NOTFOUNDS[@]}): $( implode ", " "${NOTFOUNDS[@]}" )"
 
@@ -295,83 +463,116 @@ PERF_DATA=(
     "unknown_state_containers=${#UNKNOWNS[@]};;;0;$CONTAINER_COUNTS"
 )
 
+# Compute performance data as string
+PERF_DATA_TXT="$( implode " " "${PERF_DATA[@]}" )"
+
 # Display check result message
 case $EXIT_CODE in
     0)
-		echo -n "OK - All ${#UP_TO_DATE[@]} container(s) are up-to-date"
+        message "OK - All ${#UP_TO_DATE[@]} container(s) are up-to-date |$PERF_DATA_TXT"
         ;;
     1)
-		echo -n "WARNING - ${#ERRORS[@]} container(s) need to be updated"
+        message "WARNING - ${#ERRORS[@]} container(s) need to be updated |$PERF_DATA_TXT"
         ;;
     2)
-		echo -n "CRITICAL - ${#ERRORS[@]} container(s) need to be updated"
+        message "CRITICAL - ${#ERRORS[@]} container(s) need to be updated |$PERF_DATA_TXT"
         ;;
     *)
-		echo -n "UNKNOWN - fail to retrieve status of ${#UNKNOWNS[@]} container(s)"
+        message "UNKNOWN - fail to retrieve status of ${#UNKNOWNS[@]} container(s) |$PERF_DATA_TXT"
         ;;
 esac
 
-# Add performance data
-echo " |$( implode " " "${PERF_DATA[@]}" )"
-
 # Trigger container build (if need, enabled and docker compose file is provided)
 if [ $REBUILD -eq 1 ]
 then
+    debug "Check if we have to trigger some rebuild (status file: $(rebuild_status_file))"
     if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]
     then
         debug "No upgradable container to rebuild"
-		if [ -e "$REBUILD_LOCK_FILE" ]
+        remove_rebuild_status
-		then
-			debug "Remove previous rebuild lock file ($REBUILD_LOCK_FILE)"
-			rm -f "$REBUILD_LOCK_FILE"
-		fi
     elif [ -z "$DOCKERCOMPOSE_FILE" ]
     then
-		echo
+        message
-		echo "WARNING: No docker compose file provided, can't trigger rebuild of following container(s):"
+        message "WARNING: No docker compose file provided, can't trigger rebuild of following" \
-		echo "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )"
+            "container(s):"
-	elif [ -e "$REBUILD_LOCK_FILE" ]
+        message "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )"
-	then
-		REBUILD_PID=$(cat $REBUILD_LOCK_FILE)
-		echo
-		if [ -d "/proc/$REBUILD_PID" ]
-		then
-			echo "Rebuild already triggered on $( stat --format %y "$REBUILD_LOCK_FILE" ) (PID: $REBUILD_PID)"
-			echo "You could follow it in its log file: ${REBUILD_LOG_FILE}"
     else
-			echo "Container images already rebuilt (see ${REBUILD_LOG_FILE} for details)."
+        message "Rebuilding containers:"
-			echo "You could recreate and restart them using the following command:"
+        REBUILT_CONTAINERS=()
-			echo
+        for container in ${!UPGRADABLE_CONTAINERS[@]}; do
-			echo "  $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${UPGRADABLE_CONTAINERS[@]}"
+            need_rebuild=0
-			echo
+            container_data=$( rebuild_status "$container" )
-			echo "Note:"
+            if [[ "$container_data" != "null" ]]; then
-			echo "  If new updates was published since rebuild was triggered, remove the"
+                debug "$container: data=$container_data"
-			echo "  '$REBUILD_LOCK_FILE' file and recheck it to trigger a new build."
+                trigger_date=$( jq -r .trigger_date <<< $container_data )
+                start_date=$( jq -r .start_date <<< $container_data )
+                end_date=$( jq -r .end_date <<< $container_data )
+                log=$( jq -r .log <<< $container_data )
+                if [[ "$start_date" == "null" ]]; then
+                    debug "$container: build triggered but not yet started"
+                    message "- $container: rebuild triggered on $trigger_date and not yet started"
+                elif [[ "$end_date" == "null" ]]; then
+                    debug "$container: rebuild triggered on $trigger_date and started on" \
+                        "$start_date, but not yet finish"
+                    message "- $container: rebuild triggered on $trigger_date and started on" \
+                        "$start_date, but not yet finish (log: $log)"
+                else
+                    duration=$( jq -r .duration <<< $container_data )
+                    debug "$container: rebuilt in $duration on $start_date (finish on $end_date)"
+                    prev_status=$( jq -r .status <<< $container_data )
+                    if [[ "$prev_status" == "${UPGRADABLE_CONTAINERS[$container]}" ]]; then
+                        error=$( jq -r .error <<< $container_data )
+                        if [[ "$error" != "null" ]]; then
+                            message "- $container: $error (log: $log)"
+                        else
+                            message "- $container: already rebuilt in $duration (rebuild" \
+                                "triggered on $trigger_date, started on $start_date and finish" \
+                                "at $end_date,log: $log)"
+                            REBUILT_CONTAINERS+=( $container )
                         fi
-		echo
                     else
-		echo
+                        update_rebuild_status -d "$container"
-		echo "Trigger rebuild of following container(s): ${UPGRADABLE_CONTAINERS[@]}"
+                        message "- $container: upgrade status change since last rebuild, rebuild" \
-		echo "You could follow the rebuild process in its log file: ${REBUILD_LOG_FILE}"
+                            "it again"
-		echo "You will able to recreate and restart them using the following command:"
+                        need_rebuild=1
-		echo
+                    fi
-		echo "  $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${UPGRADABLE_CONTAINERS[@]}"
+                fi
-		echo
+            else
+                debug "$container: not found in status file"
+                need_rebuild=1
+            fi
+            if [[ $need_rebuild -eq 1 ]]; then
+                update_rebuild_status "$container" \
+                    "trigger_date=$(now)" "status=${UPGRADABLE_CONTAINERS[$container]}"
+                message "- $container: rebuild triggered"
+            else
+                debug "$container: rebuild not need"
+            fi
+        done
 
-		$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache ${UPGRADABLE_CONTAINERS[@]} > $REBUILD_LOG_FILE 2>&1 &
+        # Handle rebuilt containers
-		echo $! > "$REBUILD_LOCK_FILE"
+        if [[ ${#REBUILT_CONTAINERS[@]} -gt 0 ]]; then
+            message
+            message "Some containers are ready to be recreated and restarted."
+            message "Run the following command to do it:"
+            message
+            message "  $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[@]}"
+        fi
+        message
     fi
 fi
 
 # Display details, starting by errors
 for container in ${!ERRORS[@]}
 do
-    echo ${container} - ${ERRORS[${container}]}
+    message ${container} - ${ERRORS[${container}]}
 done
 
 for container in ${!UP_TO_DATE[@]}
 do
-    echo ${container} - ${UP_TO_DATE[${container}]}
+    message ${container} - ${UP_TO_DATE[${container}]}
 done
 
 exit $EXIT_CODE
+
+# vim: shiftwidth=4 tabstop=4 expandtab