Recover disk space safely and complete metadata-only backup updates
This commit is contained in:
admin committed
1 parent
04283a607c
commit
a5a7e38853
17 files changed
+827
-209
No files matched your search
+43
-14
@@ -28,6 +28,7 @@ load_config() {
|
||||
: "${LOG_RETENTION_DAYS:=30}"
|
||||
: "${MIN_FREE_GB:=10}" "${LOCAL_RETENTION_DAYS:=7}" "${REMOTE_RETENTION_DAYS:=30}"
|
||||
: "${RETENTION_KEEP_LAST:=3}" "${ENABLE_REMOTE_RETENTION:=false}"
|
||||
: "${SPACE_KEEP_LOCAL:=1}"
|
||||
: "${ENABLE_UPLOAD:=true}" "${RCLONE_BIN:=rclone}"
|
||||
: "${RCLONE_REMOTE_BASE:=}" "${RCLONE_CONFIG:=/root/.config/rclone/rclone.conf}"
|
||||
: "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}"
|
||||
@@ -73,12 +74,13 @@ load_config() {
|
||||
[[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
|
||||
done
|
||||
(( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
|
||||
[[ $SPACE_KEEP_LOCAL =~ ^[1-9][0-9]{0,5}$ ]] || die 'SPACE_KEEP_LOCAL must be an integer between 1 and 999999'
|
||||
# Seconds/counts only: never permit disabled deadlines or unbounded retries.
|
||||
for v in RCLONE_CHECK_ATTEMPTS RCLONE_CHECK_LOW_LEVEL_RETRIES RCLONE_CHECK_TIMEOUT RCLONE_CHECK_DEADLINE RCLONE_CHECK_RETRY_DELAY; do
|
||||
[[ ${!v} =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "$v must be a bounded integer"
|
||||
done
|
||||
(( RCLONE_CHECK_ATTEMPTS >= 1 && RCLONE_CHECK_ATTEMPTS <= 10 && RCLONE_CHECK_LOW_LEVEL_RETRIES >= 1 && RCLONE_CHECK_LOW_LEVEL_RETRIES <= 20 )) || die 'Invalid readback retry limits'
|
||||
(( RCLONE_CHECK_TIMEOUT >= 1 && RCLONE_CHECK_TIMEOUT <= 3600 && RCLONE_CHECK_DEADLINE >= 1 && RCLONE_CHECK_DEADLINE <= 86400 && RCLONE_CHECK_RETRY_DELAY <= 3600 )) || die 'Invalid readback time limits'
|
||||
(( RCLONE_CHECK_ATTEMPTS >= 1 && RCLONE_CHECK_ATTEMPTS <= 10 && RCLONE_CHECK_LOW_LEVEL_RETRIES >= 1 && RCLONE_CHECK_LOW_LEVEL_RETRIES <= 20 )) || die 'Invalid remote check retry limits'
|
||||
(( RCLONE_CHECK_TIMEOUT >= 1 && RCLONE_CHECK_TIMEOUT <= 3600 && RCLONE_CHECK_DEADLINE >= 1 && RCLONE_CHECK_DEADLINE <= 86400 && RCLONE_CHECK_RETRY_DELAY <= 3600 )) || die 'Invalid remote check time limits'
|
||||
(( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
|
||||
(( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
|
||||
[[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
|
||||
@@ -277,8 +279,7 @@ check_remote() {
|
||||
for key in ${!RCLONE_CONFIG_@}; do
|
||||
[[ $key == RCLONE_CONFIG_PASS ]] || die "Remote overrides through $key are unsupported; use the config file"
|
||||
done
|
||||
# rclone reads global flags from exported RCLONE_* variables too. For example,
|
||||
# RCLONE_SIZE_ONLY bypasses byte comparison even with check --download; filters
|
||||
# rclone reads global flags from exported RCLONE_* variables too. Filters
|
||||
# can hide missing archives from both copy and check. Refuse such ambient flags
|
||||
# before the first remote operation, rather than silently weakening verification.
|
||||
# Allow only our explicit settings (passed as CLI flags) and config decryption.
|
||||
@@ -308,27 +309,26 @@ check_remote() {
|
||||
# by callers. NOTICE makes periodic progress visible without verbose HTTP logs.
|
||||
rc() { "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@"; }
|
||||
|
||||
remote_readback() {
|
||||
remote_metadata_check() {
|
||||
local attempt code=1
|
||||
# rclone check uses cmd.Run(false, ...): --retries does NOT repeat the whole
|
||||
# check. A fresh process also refreshes backend/download state after failures.
|
||||
# Always recheck every file; never turn a partial match report into success.
|
||||
# check. Retry the complete metadata comparison; never download archive data.
|
||||
# Content integrity is checked by the monthly restore test.
|
||||
for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
|
||||
echo "-- Remote readback attempt $attempt/$RCLONE_CHECK_ATTEMPTS (deadline ${RCLONE_CHECK_DEADLINE}s, idle ${RCLONE_CHECK_TIMEOUT}s)"
|
||||
echo "-- Remote metadata check attempt $attempt/$RCLONE_CHECK_ATTEMPTS (deadline ${RCLONE_CHECK_DEADLINE}s, idle ${RCLONE_CHECK_TIMEOUT}s)"
|
||||
# GNU timeout bounds backend pacing/retries as well as stalled I/O. It owns
|
||||
# a process group so TERM/KILL cannot leave an old check running on retry.
|
||||
# Multiline stats expose active checks; byte totals alone can say 100% while
|
||||
# a crypt header is still pending. Combined output names each check result.
|
||||
# Combined output names missing files and size mismatches.
|
||||
if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
|
||||
"$RCLONE_BIN" "${RCLONE_ARGS[@]}" check "$@" --download \
|
||||
"$RCLONE_BIN" "${RCLONE_ARGS[@]}" check "$@" --size-only \
|
||||
--timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" \
|
||||
--stats-one-line=false --stats-one-line-date=false --combined -; then
|
||||
echo "-- Remote readback attempt $attempt passed (all selected files)"
|
||||
echo "-- Remote metadata check attempt $attempt passed (all selected files)"
|
||||
return 0
|
||||
else
|
||||
code=$?
|
||||
fi
|
||||
echo "WARNING: Remote readback attempt $attempt/$RCLONE_CHECK_ATTEMPTS failed (exit $code; 124 means deadline exceeded)"
|
||||
echo "WARNING: Remote metadata check attempt $attempt/$RCLONE_CHECK_ATTEMPTS failed (exit $code; 124 means deadline exceeded)"
|
||||
# Preserve cancellation semantics, including a forced kill after the grace
|
||||
# period. Do not restart a command an operator/service manager has stopped.
|
||||
if (( code >= 128 )); then return "$code"; fi
|
||||
@@ -336,10 +336,39 @@ remote_readback() {
|
||||
sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?
|
||||
fi
|
||||
done
|
||||
echo 'ERROR: Remote readback exhausted; remote contents remain unverified'
|
||||
echo 'ERROR: Remote metadata check exhausted; remote file presence/sizes remain unverified'
|
||||
return "$code"
|
||||
}
|
||||
|
||||
# Before reclaiming a local copy, bind the remote sizes to the exact local
|
||||
# manifest and completion marker. No archive download or temporary disk space.
|
||||
remote_identity_check() {
|
||||
local dir=$1 remote=$2 file attempt code size
|
||||
local -a statuses
|
||||
need cmp
|
||||
for file in manifest.json COMPLETE; do
|
||||
size=$(stat -c %s -- "$dir/$file") || return 1
|
||||
code=1
|
||||
for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
|
||||
if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
|
||||
"$RCLONE_BIN" "${RCLONE_ARGS[@]}" cat "$remote/$file" --head "$((size + 1))" \
|
||||
--timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" |
|
||||
cmp -s -- "$dir/$file" -; then
|
||||
code=0
|
||||
break
|
||||
else
|
||||
statuses=("${PIPESTATUS[@]}")
|
||||
code=${statuses[0]}
|
||||
if (( code == 0 )); then code=${statuses[1]}; fi
|
||||
fi
|
||||
(( code < 128 )) || return "$code"
|
||||
echo "WARNING: Remote identity check failed: $file (attempt $attempt/$RCLONE_CHECK_ATTEMPTS)"
|
||||
if (( attempt < RCLONE_CHECK_ATTEMPTS )); then sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?; fi
|
||||
done
|
||||
(( code == 0 )) || return "$code"
|
||||
done
|
||||
}
|
||||
|
||||
# Exclusions are rooted so a nested directory named data is not accidentally lost.
|
||||
exclusions() {
|
||||
EXCLUDES=()
|
||||
|
||||
Reference in new issue
Block a user