Compare commits
2 commits
6f05b7b547
...
345316947b
Author | SHA1 | Date | |
---|---|---|---|
345316947b | |||
2b84e80148 |
3 changed files with 177 additions and 166 deletions
15
.editorconfig
Normal file
15
.editorconfig
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 4
|
||||||
|
trim_trailing_whitespace = true
|
||||||
|
insert_final_newline = true
|
||||||
|
charset = utf-8
|
||||||
|
end_of_line = lf
|
||||||
|
|
||||||
|
[*.{yaml,yml}]
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[Makefile]
|
||||||
|
indent_style = tab
|
|
@ -13,13 +13,17 @@ repos:
|
||||||
- --ignore-regex=.*codespell-ignore$
|
- --ignore-regex=.*codespell-ignore$
|
||||||
#- --write-changes # Uncomment to write changes
|
#- --write-changes # Uncomment to write changes
|
||||||
exclude_types: [csv, json]
|
exclude_types: [csv, json]
|
||||||
- repo: https://github.com/adrienverge/yamllint
|
|
||||||
rev: v1.32.0
|
|
||||||
hooks:
|
|
||||||
- id: yamllint
|
|
||||||
ignore: .github/
|
|
||||||
- repo: https://github.com/pre-commit/mirrors-prettier
|
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||||
rev: v2.7.1
|
rev: v2.7.1
|
||||||
hooks:
|
hooks:
|
||||||
- id: prettier
|
- id: prettier
|
||||||
args: ["--print-width", "100"]
|
args: ["--print-width", "100"]
|
||||||
|
- repo: https://github.com/adrienverge/yamllint
|
||||||
|
rev: v1.32.0
|
||||||
|
hooks:
|
||||||
|
- id: yamllint
|
||||||
|
ignore: .github/
|
||||||
|
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||||
|
rev: v0.10.0.1
|
||||||
|
hooks:
|
||||||
|
- id: shellcheck
|
||||||
|
|
|
@ -16,7 +16,7 @@ MAX_PARALLEL_CHECKS=4
|
||||||
ONLY_CONTAINERS=()
|
ONLY_CONTAINERS=()
|
||||||
EXCLUDED_CONTAINERS=( buildx_buildkit_default )
|
EXCLUDED_CONTAINERS=( buildx_buildkit_default )
|
||||||
REBUILD=0
|
REBUILD=0
|
||||||
REBUILD_DATA_DIR="/var/log/$(basename $0)"
|
REBUILD_DATA_DIR="/var/log/$(basename "$0")"
|
||||||
REBUILD_CRON=0
|
REBUILD_CRON=0
|
||||||
DEPLOY_CRON=0
|
DEPLOY_CRON=0
|
||||||
declare -rA CHECK_PLUGINS=(
|
declare -rA CHECK_PLUGINS=(
|
||||||
|
@ -24,38 +24,43 @@ declare -rA CHECK_PLUGINS=(
|
||||||
["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk"
|
["/usr/lib/nagios/plugins/check_apk"]="/usr/lib/nagios/plugins/check_apk"
|
||||||
)
|
)
|
||||||
|
|
||||||
function now() { [[ -z "$1" ]] && date "+%F %H:%M:%S" || date -d "@$1" "+%F %H:%M:%S" ; }
|
function now() {
|
||||||
|
if [[ -z "$1" ]]; then date "+%F %H:%M:%S"; else date -d "@$1" "+%F %H:%M:%S"; fi
|
||||||
|
}
|
||||||
function current_time() { date "+%s"; }
|
function current_time() { date "+%s"; }
|
||||||
|
|
||||||
function debug() {
|
function debug() {
|
||||||
[[ $DEBUG -eq 0 ]] && return
|
[[ $DEBUG -eq 0 ]] && return
|
||||||
if [[ -n "$LOG_FILE" ]] && [[ $CONSOLE -eq 1 ]]; then
|
if [[ -n "$LOG_FILE" ]] && [[ $CONSOLE -eq 1 ]]; then
|
||||||
echo -e "$(now) - [DEBUG] $@" | tee -a "$LOG_FILE" 2>&1
|
echo -e "$(now) - [DEBUG] $*" | tee -a "$LOG_FILE" 2>&1
|
||||||
elif [[ -n "$LOG_FILE" ]]; then
|
elif [[ -n "$LOG_FILE" ]]; then
|
||||||
echo -e "$(now) - [DEBUG] $@" >> "$LOG_FILE"
|
echo -e "$(now) - [DEBUG] $*" >> "$LOG_FILE"
|
||||||
else
|
else
|
||||||
>&2 echo -e "[DEBUG] $@"
|
>&2 echo -e "[DEBUG] $*"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function log() {
|
function log() {
|
||||||
[[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${@:2}" >> "$LOG_FILE"
|
[[ -n "$LOG_FILE" ]] && echo -e "$(now) - [$1] ${*:2}" >> "$LOG_FILE"
|
||||||
[[ "$1" == "ERROR" ]] && >&2 echo -e "ERROR - ${@:2}" || echo -e "${@:2}"
|
if [[ "$1" == "ERROR" ]]; then
|
||||||
|
>&2 echo -e "ERROR - ${*:2}"
|
||||||
|
else
|
||||||
|
echo -e "${*:2}"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function message() { log INFO $@ ; }
|
function message() { log INFO "$@" ; }
|
||||||
function error() { log ERROR $@ ; }
|
function error() { log ERROR "$@" ; }
|
||||||
|
|
||||||
function is_empty() {
|
function is_empty() {
|
||||||
[ $# -gt 0 ] && return 1
|
[[ $# -gt 0 ]] && return 1
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
function in_array() {
|
function in_array() {
|
||||||
param=$1;
|
local param=$1 elem;
|
||||||
shift;
|
shift;
|
||||||
for elem in "$@";
|
for elem in "$@"; do
|
||||||
do
|
|
||||||
[[ "$param" = "$elem" ]] && return 0;
|
[[ "$param" = "$elem" ]] && return 0;
|
||||||
done;
|
done;
|
||||||
return 1
|
return 1
|
||||||
|
@ -69,13 +74,13 @@ function implode() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function format_duration {
|
function format_duration {
|
||||||
local T=$1
|
local t=$1
|
||||||
local D=$((T/60/60/24))
|
local d=$((t/60/60/24))
|
||||||
local H=$((T/60/60%24))
|
local h=$((t/60/60%24))
|
||||||
local M=$((T/60%60))
|
local m=$((t/60%60))
|
||||||
local S=$((T%60))
|
local s=$((t%60))
|
||||||
(( $D > 0 )) && printf '%d days and ' $D
|
[[ $d -gt 0 ]] && printf '%d days and ' $d
|
||||||
printf '%02d:%02d:%02d' $H $M $S
|
printf '%02d:%02d:%02d' $h $m $s
|
||||||
}
|
}
|
||||||
|
|
||||||
REBUILD_STATUS_FILE=""
|
REBUILD_STATUS_FILE=""
|
||||||
|
@ -88,54 +93,61 @@ function rebuild_status_file() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function rebuild_status() {
|
function rebuild_status() {
|
||||||
if [[ -z "$1" ]]; then
|
local output_var=$1
|
||||||
cat "$(rebuild_status_file)"
|
[[ -n "$REBUILD_STATUS_FILE" ]] || rebuild_status_file
|
||||||
|
if [[ -z "$2" ]]; then
|
||||||
|
declare -g "$output_var=$( cat "$REBUILD_STATUS_FILE" )"
|
||||||
else
|
else
|
||||||
rebuild_status | jq -r --arg container "$1" '.[$container]'
|
declare -g "$output_var=$(
|
||||||
|
jq -r --arg container "$1" '.[$container]' < "$REBUILD_STATUS_FILE"
|
||||||
|
)"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function update_rebuild_status() {
|
function update_rebuild_status() {
|
||||||
|
local data
|
||||||
if [[ "$1" == "-d" ]]; then
|
if [[ "$1" == "-d" ]]; then
|
||||||
local data=$(
|
rebuild_status DATA
|
||||||
rebuild_status | jq \
|
# shellcheck disable=SC2153
|
||||||
|
data=$(
|
||||||
|
jq \
|
||||||
--arg container "$2" \
|
--arg container "$2" \
|
||||||
'del(.[$container])'
|
'del(.[$container])' <<< "$DATA"
|
||||||
)
|
)
|
||||||
else
|
else
|
||||||
local args=( --arg container "$1" )
|
local args=( --arg container "$1" ) arg name value expr=( )
|
||||||
local expr=( )
|
|
||||||
for arg in "${@:2}"; do
|
for arg in "${@:2}"; do
|
||||||
local name=$( echo "$arg" | cut -d'=' -f1 )
|
name=$( cut -d'=' -f1 <<< "$arg" )
|
||||||
local value=$( echo "$arg" | sed 's/[^=]\+=//' )
|
# shellcheck disable=SC2001
|
||||||
|
value=$( sed 's/[^=]\+=//' <<< "$arg" )
|
||||||
args+=( --arg "$name" "$value" )
|
args+=( --arg "$name" "$value" )
|
||||||
expr+=( ".[\$container].$name=\$$name" )
|
expr+=( ".[\$container].$name=\$$name" )
|
||||||
done
|
done
|
||||||
local data=$(
|
rebuild_status DATA
|
||||||
rebuild_status | jq \
|
data=$(
|
||||||
"${args[@]}" "$( implode ' | ' "${expr[@]}" )"
|
jq "${args[@]}" "$( implode ' | ' "${expr[@]}" )" <<< "$DATA"
|
||||||
)
|
)
|
||||||
fi
|
fi
|
||||||
cat <<< $data > $(rebuild_status_file)
|
cat <<< "$data" > "$REBUILD_STATUS_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
function remove_rebuild_status() {
|
function remove_rebuild_status() {
|
||||||
rebuild_status_file > /dev/null
|
[[ -n "$REBUILD_STATUS_FILE" ]] || rebuild_status_file
|
||||||
[[ -e "$REBUILD_STATUS_FILE" ]] || return 0
|
[[ -e "$REBUILD_STATUS_FILE" ]] || return 0
|
||||||
debug "Remove previous rebuild status file ($REBUILD_STATUS_FILE) and log files it contains"
|
debug "Remove previous rebuild status file ($REBUILD_STATUS_FILE) and log files it contains"
|
||||||
for log in $( jq -r '.[] | .log' $REBUILD_STATUS_FILE ); do
|
for log in $( jq -r '.[] | .log' "$REBUILD_STATUS_FILE" ); do
|
||||||
debug " remove old container log file $log"
|
debug " remove old container log file $log"
|
||||||
rm -f $log
|
rm -f "$log"
|
||||||
done
|
done
|
||||||
debug " remove status file"
|
debug " remove status file"
|
||||||
rm -f "$REBUILD_STATUS_FILE"
|
rm -f "$REBUILD_STATUS_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
function usage() {
|
function usage() {
|
||||||
error="$1"
|
local error="$1"
|
||||||
[ -n "$error" ] && message "$error"
|
[[ -n "$error" ]] && message "$error"
|
||||||
cat << EOF
|
cat << EOF
|
||||||
Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
|
Usage : $(basename "$0") [-d] [-E /path/to/engine] [container1,...]
|
||||||
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]},
|
-E [path] Force a specific engine (possible values: ${POSSIBLE_ENGINES[@]},
|
||||||
default: $ENGINE)
|
default: $ENGINE)
|
||||||
-x [container] Exclude specified container (could be repeat)
|
-x [container] Exclude specified container (could be repeat)
|
||||||
|
@ -156,13 +168,12 @@ Usage : $(basename $0) [-d] [-E /path/to/engine] [container1,...]
|
||||||
-X Enable bash tracing (=set -x)
|
-X Enable bash tracing (=set -x)
|
||||||
-h Show this message
|
-h Show this message
|
||||||
EOF
|
EOF
|
||||||
[ -n "$error" ] && exit 1
|
[[ -n "$error" ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
||||||
idx=1
|
idx=1
|
||||||
while [ $idx -le $# ]
|
while [[ $idx -le $# ]]; do
|
||||||
do
|
|
||||||
OPT=${!idx}
|
OPT=${!idx}
|
||||||
case $OPT in
|
case $OPT in
|
||||||
-d)
|
-d)
|
||||||
|
@ -181,9 +192,8 @@ do
|
||||||
-E)
|
-E)
|
||||||
((idx++))
|
((idx++))
|
||||||
ENGINE=${!idx}
|
ENGINE=${!idx}
|
||||||
if [ ! -x "$ENGINE" ]
|
if [[ ! -x "$ENGINE" ]]; then
|
||||||
then
|
in_array "$ENGINE" "${POSSIBLE_ENGINES[@]}" || usage "Invalid engine $ENGINE"
|
||||||
in_array $ENGINE ${POSSIBLE_ENGINES[@]} || usage "Invalid engine $ENGINE"
|
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
-f)
|
-f)
|
||||||
|
@ -199,7 +209,7 @@ do
|
||||||
;;
|
;;
|
||||||
-x)
|
-x)
|
||||||
((idx++))
|
((idx++))
|
||||||
EXCLUDED_CONTAINERS+=( ${!idx} )
|
EXCLUDED_CONTAINERS+=( "${!idx}" )
|
||||||
;;
|
;;
|
||||||
-M)
|
-M)
|
||||||
((idx++))
|
((idx++))
|
||||||
|
@ -215,33 +225,27 @@ do
|
||||||
DEPLOY_CRON=1
|
DEPLOY_CRON=1
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
ONLY_CONTAINERS+=( $OPT )
|
ONLY_CONTAINERS+=( "$OPT" )
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
((idx++))
|
((idx++))
|
||||||
done
|
done
|
||||||
|
|
||||||
debug "Start with parameters: $@"
|
debug "Start with parameters: $*"
|
||||||
|
|
||||||
! is_empty $ONLY_CONTAINERS && debug "Only containers: ${ONLY_CONTAINERS[@]}"
|
is_empty "${ONLY_CONTAINERS[@]}" || debug "Only containers: ${ONLY_CONTAINERS[*]}"
|
||||||
|
|
||||||
if [ "$ENGINE" == "auto" ]
|
if [[ "$ENGINE" == "auto" ]]; then
|
||||||
then
|
|
||||||
debug "Auto-detect engine..."
|
debug "Auto-detect engine..."
|
||||||
for engine in ${POSSIBLE_ENGINES[@]}
|
for engine in "${POSSIBLE_ENGINES[@]}"; do
|
||||||
do
|
[[ "$engine" == "auto" ]] && continue
|
||||||
[ "$engine" == "auto" ] && continue
|
if which "$engine" > /dev/null 2>&1; then
|
||||||
which "$engine" > /dev/null 2>&1
|
|
||||||
if [ $? -ne 0 ]
|
|
||||||
then
|
|
||||||
debug "$engine not found"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
ENGINE="$engine"
|
ENGINE="$engine"
|
||||||
break
|
break
|
||||||
|
fi
|
||||||
|
debug "$engine not found"
|
||||||
done
|
done
|
||||||
if [ -z "$ENGINE" ]
|
if [[ -z "$ENGINE" ]]; then
|
||||||
then
|
|
||||||
message "UNKNOWN - Fail to auto-detect engine"
|
message "UNKNOWN - Fail to auto-detect engine"
|
||||||
exit 3
|
exit 3
|
||||||
fi
|
fi
|
||||||
|
@ -260,29 +264,29 @@ if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ $REBUILD_CRON -eq 1 ]]; then
|
if [[ $REBUILD_CRON -eq 1 ]]; then
|
||||||
to_rebuild=(
|
rebuild_status DATA
|
||||||
$(
|
mapfile -t to_rebuild < <(
|
||||||
rebuild_status | jq -r -c \
|
jq -r -c 'to_entries[] | select(
|
||||||
'to_entries[] | select((.value.start_date|not) and (.value.error|not)) | .key'
|
(.value.start_date|not) and (.value.error|not)
|
||||||
)
|
) | .key' <<< "$DATA"
|
||||||
)
|
)
|
||||||
if [[ ${#to_rebuild[@]} -eq 0 ]]; then
|
if [[ ${#to_rebuild[@]} -eq 0 ]]; then
|
||||||
debug "No container need to be rebuild"
|
debug "No container need to be rebuild"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[@]}"
|
message "${#to_rebuild[@]} container(s) to rebuild: ${to_rebuild[*]}"
|
||||||
error=0
|
error=0
|
||||||
for container in ${to_rebuild[@]}; do
|
for container in "${to_rebuild[@]}"; do
|
||||||
log="$REBUILD_DATA_DIR/$container.log"
|
log="$REBUILD_DATA_DIR/$container.log"
|
||||||
message " $container: start building image (log=$log)"
|
message " $container: start building image (log=$log)"
|
||||||
start_time=$(current_time)
|
start_time=$(current_time)
|
||||||
update_rebuild_status "$container" "start_date=$(now $start_time)" "log=$log"
|
update_rebuild_status "$container" "start_date=$(now "$start_time")" "log=$log"
|
||||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE build --no-cache $container >> $log 2>&1
|
"$COMPOSE_BIN" -f "$DOCKERCOMPOSE_FILE" build --no-cache "$container" >> "$log" 2>&1
|
||||||
result=$?
|
result=$?
|
||||||
end_time=$(current_time)
|
end_time=$(current_time)
|
||||||
(( duration=end_time-start_time ))
|
(( duration=end_time-start_time ))
|
||||||
duration=$(format_duration $duration)
|
duration=$(format_duration "$duration")
|
||||||
container_info=( "end_date=$(now $end_time)" "duration=$duration" )
|
container_info=( "end_date=$(now "$end_time")" "duration=$duration" )
|
||||||
if [[ $result -eq 0 ]]; then
|
if [[ $result -eq 0 ]]; then
|
||||||
message " $container: rebuilt in $duration"
|
message " $container: rebuilt in $duration"
|
||||||
else
|
else
|
||||||
|
@ -296,26 +300,24 @@ if [[ $REBUILD_CRON -eq 1 ]]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ $DEPLOY_CRON -eq 1 ]]; then
|
if [[ $DEPLOY_CRON -eq 1 ]]; then
|
||||||
to_deploy=(
|
rebuild_status DATA
|
||||||
$(
|
mapfile -t to_deploy < <(
|
||||||
rebuild_status | jq -r -c \
|
jq -r -c \
|
||||||
'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key'
|
'to_entries[] | select((.value.end_date) and (.value.error|not)) | .key' <<< "$DATA"
|
||||||
)
|
|
||||||
)
|
)
|
||||||
if [[ ${#to_deploy[@]} -eq 0 ]]; then
|
if [[ ${#to_deploy[@]} -eq 0 ]]; then
|
||||||
debug "No container need to be deploy"
|
debug "No container need to be deploy"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[@]}"
|
message "${#to_deploy[@]} container(s) to deploy: ${to_deploy[*]}"
|
||||||
error=0
|
error=0
|
||||||
for container in ${to_deploy[@]}; do
|
for container in "${to_deploy[@]}"; do
|
||||||
message " $container: deploying..."
|
message " $container: deploying..."
|
||||||
log="$REBUILD_DATA_DIR/$container.log"
|
log="$REBUILD_DATA_DIR/$container.log"
|
||||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps $container > $log 2>&1
|
if $COMPOSE_BIN -f "$DOCKERCOMPOSE_FILE" up -d --no-deps "$container" > "$log" 2>&1; then
|
||||||
if [[ $? -eq 0 ]]; then
|
|
||||||
message " $container: done"
|
message " $container: done"
|
||||||
update_rebuild_status -d "$container"
|
update_rebuild_status -d "$container"
|
||||||
rm -f $log
|
rm -f "$log"
|
||||||
else
|
else
|
||||||
error " $container: fail to deploy new container image"
|
error " $container: fail to deploy new container image"
|
||||||
update_rebuild_status "$container" "error=fail to deploy new container image"
|
update_rebuild_status "$container" "error=fail to deploy new container image"
|
||||||
|
@ -331,113 +333,106 @@ declare -A CONTAINER_STATUS_FILE
|
||||||
declare -A CONTAINER_PID
|
declare -A CONTAINER_PID
|
||||||
declare -A UP_TO_DATE
|
declare -A UP_TO_DATE
|
||||||
declare -A ERRORS
|
declare -A ERRORS
|
||||||
declare -A UNKNOWNS
|
declare -a UNKNOWNS
|
||||||
declare -A UPGRADABLE_CONTAINERS
|
declare -A UPGRADABLE_CONTAINERS
|
||||||
CHECKED_CONTAINERS=( )
|
CHECKED_CONTAINERS=( )
|
||||||
|
|
||||||
debug "List running containers..."
|
debug "List running containers..."
|
||||||
[[ -n "$DOCKERCOMPOSE_FILE" ]] && \
|
if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
|
||||||
RUNNING_CONTAINERS=$($COMPOSE_BIN -f $DOCKERCOMPOSE_FILE ps --format '{{.Service}}' | tr '\n' ' ') ||
|
RUNNING_CONTAINERS=$(
|
||||||
RUNNING_CONTAINERS=$($ENGINE ps --format '{{.Names}}' | tr '\n' ' ')
|
$COMPOSE_BIN -f "$DOCKERCOMPOSE_FILE" ps --format '{{.Service}}' | tr '\n' ' '
|
||||||
|
)
|
||||||
|
else
|
||||||
|
RUNNING_CONTAINERS=$( $ENGINE ps --format '{{.Names}}' | tr '\n' ' ' )
|
||||||
|
fi
|
||||||
debug "Running containers: $RUNNING_CONTAINERS"
|
debug "Running containers: $RUNNING_CONTAINERS"
|
||||||
|
|
||||||
function exec_in_container() {
|
function exec_in_container() {
|
||||||
container=$1
|
local container=$1
|
||||||
shift;
|
shift;
|
||||||
if [ -n "$DOCKERCOMPOSE_FILE" ]
|
if [[ -n "$DOCKERCOMPOSE_FILE" ]]; then
|
||||||
then
|
$COMPOSE_BIN -f "$DOCKERCOMPOSE_FILE" exec "$container" "$@"
|
||||||
$COMPOSE_BIN -f $DOCKERCOMPOSE_FILE exec $container $@
|
|
||||||
return $?
|
return $?
|
||||||
fi
|
fi
|
||||||
$ENGINE exec $container $@
|
$ENGINE exec "$container" "$@"
|
||||||
return $?
|
return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
# Implement check inside a function to allow running it in parallel
|
# Implement check inside a function to allow running it in parallel
|
||||||
# Parameters : [container] [output file]
|
# Parameters : [container] [output file]
|
||||||
function check_container() {
|
function check_container() {
|
||||||
container="$1"
|
local container="$1" output_file="$2" status="" check_plugin check_plugin_cmd=() status ex
|
||||||
output_file="$2"
|
for check_plugin in "${!CHECK_PLUGINS[@]}"; do
|
||||||
STATUS=""
|
if ! exec_in_container "$container" test -e "$check_plugin" > /dev/null 2>&1; then
|
||||||
for check_plugin in ${CHECK_PLUGINS[@]}
|
|
||||||
do
|
|
||||||
exec_in_container $container test -e $check_plugin > /dev/null 2>&1
|
|
||||||
if [ $? -ne 0 ]
|
|
||||||
then
|
|
||||||
debug "$container - Plugin $check_plugin not found"
|
debug "$container - Plugin $check_plugin not found"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
debug "$container - Plugin $check_plugin found, use it"
|
debug "$container - Plugin $check_plugin found, use it"
|
||||||
STATUS="$(exec_in_container $container ${CHECK_PLUGINS[${check_plugin}]} 2>&1)"
|
read -ra check_plugin_cmd <<< "${CHECK_PLUGINS[${check_plugin}]}"
|
||||||
|
status="$( exec_in_container "$container" "${check_plugin_cmd[@]}" 2>&1 )"
|
||||||
ex=$?
|
ex=$?
|
||||||
debug "$container - Plugin output: $STATUS"
|
debug "$container - Plugin output: $status"
|
||||||
debug "$container - Plugin exit code: $ex"
|
debug "$container - Plugin exit code: $ex"
|
||||||
break
|
break
|
||||||
done
|
done
|
||||||
if [ -z "$STATUS" ]
|
if [[ -z "$status" ]]; then
|
||||||
then
|
|
||||||
debug "$container - No check plugin found"
|
debug "$container - No check plugin found"
|
||||||
STATUS="UNKNOWN - No check plugin available"
|
status="UNKNOWN - No check plugin available"
|
||||||
ex=3
|
ex=3
|
||||||
fi
|
fi
|
||||||
echo $STATUS > $output_file
|
echo -e "$status" > "$output_file"
|
||||||
return $ex
|
return $ex
|
||||||
}
|
}
|
||||||
|
|
||||||
debug "Trigger check of all selected containers..."
|
debug "Trigger check of all selected containers..."
|
||||||
for container in $RUNNING_CONTAINERS
|
for container in $RUNNING_CONTAINERS; do
|
||||||
do
|
if ! is_empty "${ONLY_CONTAINERS[@]}" && ! in_array "$container" "${ONLY_CONTAINERS[@]}"; then
|
||||||
if ! is_empty $ONLY_CONTAINERS && ! in_array $container ${ONLY_CONTAINERS[@]}
|
|
||||||
then
|
|
||||||
debug "$container - Ignored"
|
debug "$container - Ignored"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
if in_array $container ${EXCLUDED_CONTAINERS[@]}
|
if in_array "$container" "${EXCLUDED_CONTAINERS[@]}"; then
|
||||||
then
|
|
||||||
debug "$container - Excluded"
|
debug "$container - Excluded"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
if [ $MAX_PARALLEL_CHECKS -gt 0 -a "$(jobs | wc -l)" -ge $MAX_PARALLEL_CHECKS ]
|
if [[ "$MAX_PARALLEL_CHECKS" -gt 0 ]] && \
|
||||||
then
|
[[ "$(jobs | wc -l)" -ge "$MAX_PARALLEL_CHECKS" ]]; then
|
||||||
debug "Max parallel checks count reached. Waiting some check ending"
|
debug "Max parallel checks count reached. Waiting some check ending"
|
||||||
wait -n
|
wait -n
|
||||||
debug "Some check ended, continue"
|
debug "Some check ended, continue"
|
||||||
fi
|
fi
|
||||||
CHECKED_CONTAINERS+=( "$container" )
|
CHECKED_CONTAINERS+=( "$container" )
|
||||||
CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) )
|
CONTAINER_STATUS_FILE+=( ["$container"]=$( mktemp ) )
|
||||||
check_container $container ${CONTAINER_STATUS_FILE[$container]} & CONTAINER_PID+=( ["$container"]=$! )
|
check_container "$container" "${CONTAINER_STATUS_FILE[$container]}" & CONTAINER_PID+=( ["$container"]=$! )
|
||||||
done
|
done
|
||||||
|
|
||||||
debug "Wait for each individual container check and handle their result..."
|
debug "Wait for each individual container check and handle their result..."
|
||||||
for container in ${!CONTAINER_PID[@]}
|
for container in "${!CONTAINER_PID[@]}"; do
|
||||||
do
|
|
||||||
pid=${CONTAINER_PID[$container]}
|
pid=${CONTAINER_PID[$container]}
|
||||||
debug "$container - Waiting for PID ${pid}..."
|
debug "$container - Waiting for PID ${pid}..."
|
||||||
wait $pid
|
wait "$pid"
|
||||||
ex=$?
|
ex=$?
|
||||||
debug "$container - Check return ${ex}"
|
debug "$container - Check return ${ex}"
|
||||||
STATUS=$( cat ${CONTAINER_STATUS_FILE[$container]} )
|
STATUS=$( cat "${CONTAINER_STATUS_FILE[$container]}" )
|
||||||
rm -f ${CONTAINER_STATUS_FILE[$container]}
|
rm -f "${CONTAINER_STATUS_FILE[$container]}"
|
||||||
if [ $ex -eq 0 ]
|
if [[ $ex -eq 0 ]]; then
|
||||||
then
|
|
||||||
UP_TO_DATE+=( ["$container"]=$STATUS )
|
UP_TO_DATE+=( ["$container"]=$STATUS )
|
||||||
else
|
else
|
||||||
ERRORS+=( ["$container"]=$STATUS )
|
ERRORS+=( ["$container"]=$STATUS )
|
||||||
[ $ex -ge 3 ] && UNKNOWNS+=( "$container" ) || \
|
if [[ $ex -ge 3 ]]; then
|
||||||
|
UNKNOWNS+=( "$container" )
|
||||||
|
else
|
||||||
UPGRADABLE_CONTAINERS+=( ["$container"]="$STATUS" )
|
UPGRADABLE_CONTAINERS+=( ["$container"]="$STATUS" )
|
||||||
fi
|
fi
|
||||||
[ $EXIT_CODE -ge $ex ] && continue
|
fi
|
||||||
[ $ex -gt 3 ] && ex=3
|
[[ $EXIT_CODE -ge $ex ]] && continue
|
||||||
|
[[ $ex -gt 3 ]] && ex=3
|
||||||
EXIT_CODE=$ex
|
EXIT_CODE=$ex
|
||||||
done
|
done
|
||||||
|
|
||||||
NOTFOUNDS=()
|
NOTFOUNDS=()
|
||||||
if ! is_empty $ONLY_CONTAINERS
|
if ! is_empty "${ONLY_CONTAINERS[@]}"; then
|
||||||
then
|
for container in "${ONLY_CONTAINERS[@]}"; do
|
||||||
for container in ${ONLY_CONTAINERS[@]}
|
if ! in_array "$container" "${CHECKED_CONTAINERS[@]}"; then
|
||||||
do
|
|
||||||
if ! in_array $container ${CHECKED_CONTAINERS[@]}
|
|
||||||
then
|
|
||||||
debug "$container - Container not found"
|
debug "$container - Container not found"
|
||||||
ERRORS+=( ["$container"]="Container not found" )
|
ERRORS+=( ["$container"]="Container not found" )
|
||||||
NOTFOUNDS+=( "$container" )
|
NOTFOUNDS+=( "$container" )
|
||||||
|
@ -455,7 +450,7 @@ debug "Containers with errors (${#ERRORS[@]}): $( implode ", " "${!ERRORS[@]}" )
|
||||||
debug "Not found containers (${#NOTFOUNDS[@]}): $( implode ", " "${NOTFOUNDS[@]}" )"
|
debug "Not found containers (${#NOTFOUNDS[@]}): $( implode ", " "${NOTFOUNDS[@]}" )"
|
||||||
|
|
||||||
# Compute performance data
|
# Compute performance data
|
||||||
let CONTAINER_COUNTS=${#CHECKED_CONTAINERS[@]}+${#NOTFOUNDS[@]}
|
(( CONTAINER_COUNTS=${#CHECKED_CONTAINERS[@]}+${#NOTFOUNDS[@]} ))
|
||||||
PERF_DATA=(
|
PERF_DATA=(
|
||||||
"uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS"
|
"uptodate_containers=${#UP_TO_DATE[@]};;;0;$CONTAINER_COUNTS"
|
||||||
"upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS"
|
"upgradable_containers=${#UPGRADABLE_CONTAINERS[@]};;;0;$CONTAINER_COUNTS"
|
||||||
|
@ -483,31 +478,30 @@ case $EXIT_CODE in
|
||||||
esac
|
esac
|
||||||
|
|
||||||
# Trigger container build (if need, enabled and docker compose file is provided)
|
# Trigger container build (if need, enabled and docker compose file is provided)
|
||||||
if [ $REBUILD -eq 1 ]
|
if [[ $REBUILD -eq 1 ]]; then
|
||||||
then
|
[[ -n "$REBUILD_STATUS_FILE" ]] || rebuild_status_file
|
||||||
debug "Check if we have to trigger some rebuild (status file: $(rebuild_status_file))"
|
debug "Check if we have to trigger some rebuild (status file: $REBUILD_STATUS_FILE)"
|
||||||
if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]
|
if [ ${#UPGRADABLE_CONTAINERS[@]} -eq 0 ]; then
|
||||||
then
|
|
||||||
debug "No upgradable container to rebuild"
|
debug "No upgradable container to rebuild"
|
||||||
remove_rebuild_status
|
remove_rebuild_status
|
||||||
elif [ -z "$DOCKERCOMPOSE_FILE" ]
|
elif [[ -z "$DOCKERCOMPOSE_FILE" ]]; then
|
||||||
then
|
|
||||||
message
|
message
|
||||||
message "WARNING: No docker compose file provided, can't trigger rebuild of following" \
|
message "WARNING: No docker compose file provided, can't trigger rebuild of following" \
|
||||||
"container(s):"
|
"container(s):"
|
||||||
message "- $( implode "\n- " ${UPGRADABLE_CONTAINERS[@]} )"
|
message "- $( implode "\n- " "${UPGRADABLE_CONTAINERS[@]}" )"
|
||||||
else
|
else
|
||||||
message "Rebuilding containers:"
|
message "Rebuilding containers:"
|
||||||
REBUILT_CONTAINERS=()
|
REBUILT_CONTAINERS=()
|
||||||
for container in ${!UPGRADABLE_CONTAINERS[@]}; do
|
for container in "${!UPGRADABLE_CONTAINERS[@]}"; do
|
||||||
need_rebuild=0
|
need_rebuild=0
|
||||||
container_data=$( rebuild_status "$container" )
|
rebuild_status "$container" CONTAINER_DATA
|
||||||
if [[ "$container_data" != "null" ]]; then
|
# shellcheck disable=SC2153
|
||||||
debug "$container: data=$container_data"
|
if [[ "$CONTAINER_DATA" != "null" ]]; then
|
||||||
trigger_date=$( jq -r .trigger_date <<< $container_data )
|
debug "$container: data=$CONTAINER_DATA"
|
||||||
start_date=$( jq -r .start_date <<< $container_data )
|
trigger_date=$( jq -r .trigger_date <<< "$CONTAINER_DATA" )
|
||||||
end_date=$( jq -r .end_date <<< $container_data )
|
start_date=$( jq -r .start_date <<< "$CONTAINER_DATA" )
|
||||||
log=$( jq -r .log <<< $container_data )
|
end_date=$( jq -r .end_date <<< "$CONTAINER_DATA" )
|
||||||
|
log=$( jq -r .log <<< "$CONTAINER_DATA" )
|
||||||
if [[ "$start_date" == "null" ]]; then
|
if [[ "$start_date" == "null" ]]; then
|
||||||
debug "$container: build triggered but not yet started"
|
debug "$container: build triggered but not yet started"
|
||||||
message "- $container: rebuild triggered on $trigger_date and not yet started"
|
message "- $container: rebuild triggered on $trigger_date and not yet started"
|
||||||
|
@ -517,18 +511,18 @@ then
|
||||||
message "- $container: rebuild triggered on $trigger_date and started on" \
|
message "- $container: rebuild triggered on $trigger_date and started on" \
|
||||||
"$start_date, but not yet finish (log: $log)"
|
"$start_date, but not yet finish (log: $log)"
|
||||||
else
|
else
|
||||||
duration=$( jq -r .duration <<< $container_data )
|
duration=$( jq -r .duration <<< "$CONTAINER_DATA" )
|
||||||
debug "$container: rebuilt in $duration on $start_date (finish on $end_date)"
|
debug "$container: rebuilt in $duration on $start_date (finish on $end_date)"
|
||||||
prev_status=$( jq -r .status <<< $container_data )
|
prev_status=$( jq -r .status <<< "$CONTAINER_DATA" )
|
||||||
if [[ "$prev_status" == "${UPGRADABLE_CONTAINERS[$container]}" ]]; then
|
if [[ "$prev_status" == "${UPGRADABLE_CONTAINERS[$container]}" ]]; then
|
||||||
error=$( jq -r .error <<< $container_data )
|
error=$( jq -r .error <<< "$CONTAINER_DATA" )
|
||||||
if [[ "$error" != "null" ]]; then
|
if [[ "$error" != "null" ]]; then
|
||||||
message "- $container: $error (log: $log)"
|
message "- $container: $error (log: $log)"
|
||||||
else
|
else
|
||||||
message "- $container: already rebuilt in $duration (rebuild" \
|
message "- $container: already rebuilt in $duration (rebuild" \
|
||||||
"triggered on $trigger_date, started on $start_date and finish" \
|
"triggered on $trigger_date, started on $start_date and finish" \
|
||||||
"at $end_date,log: $log)"
|
"at $end_date,log: $log)"
|
||||||
REBUILT_CONTAINERS+=( $container )
|
REBUILT_CONTAINERS+=( "$container" )
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
update_rebuild_status -d "$container"
|
update_rebuild_status -d "$container"
|
||||||
|
@ -556,21 +550,19 @@ then
|
||||||
message "Some containers are ready to be recreated and restarted."
|
message "Some containers are ready to be recreated and restarted."
|
||||||
message "Run the following command to do it:"
|
message "Run the following command to do it:"
|
||||||
message
|
message
|
||||||
message " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[@]}"
|
message " $COMPOSE_BIN -f $DOCKERCOMPOSE_FILE up -d --no-deps ${REBUILT_CONTAINERS[*]}"
|
||||||
fi
|
fi
|
||||||
message
|
message
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Display details, starting by errors
|
# Display details, starting by errors
|
||||||
for container in ${!ERRORS[@]}
|
for container in "${!ERRORS[@]}"; do
|
||||||
do
|
message "${container}" - "${ERRORS[${container}]}"
|
||||||
message ${container} - ${ERRORS[${container}]}
|
|
||||||
done
|
done
|
||||||
|
|
||||||
for container in ${!UP_TO_DATE[@]}
|
for container in "${!UP_TO_DATE[@]}"; do
|
||||||
do
|
message "${container}" - "${UP_TO_DATE[${container}]}"
|
||||||
message ${container} - ${UP_TO_DATE[${container}]}
|
|
||||||
done
|
done
|
||||||
|
|
||||||
exit $EXIT_CODE
|
exit $EXIT_CODE
|
||||||
|
|
Loading…
Reference in a new issue