Bound and retry complete remote readback checks

lib/common.sh

@@ -33,6 +33,8 @@ load_config() {
   : "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}"
   : "${RCLONE_RETRIES_SLEEP:=30s}" "${RCLONE_TIMEOUT:=1h}" "${RCLONE_CONTIMEOUT:=30s}"
   : "${RCLONE_STATS_INTERVAL:=1m}"
+  : "${RCLONE_CHECK_ATTEMPTS:=3}" "${RCLONE_CHECK_RETRY_DELAY:=30}"
+  : "${RCLONE_CHECK_TIMEOUT:=300}" "${RCLONE_CHECK_DEADLINE:=3600}" "${RCLONE_CHECK_LOW_LEVEL_RETRIES:=3}"
   : "${RCLONE_TRANSFERS:=2}" "${RCLONE_CHECKERS:=4}" "${RCLONE_ONEDRIVE_CHUNK_SIZE:=80Mi}"
   : "${RCLONE_BWLIMIT:=0}" "${ENABLE_MAIL_REPORT:=true}" "${MAIL_TO:=johannes}"
   # Keep the old report settings; an empty sender uses app-backup@<FQDN>.
@@ -71,6 +73,12 @@ load_config() {
     [[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
   done
   (( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
+  # Seconds/counts only: never permit disabled deadlines or unbounded retries.
+  for v in RCLONE_CHECK_ATTEMPTS RCLONE_CHECK_LOW_LEVEL_RETRIES RCLONE_CHECK_TIMEOUT RCLONE_CHECK_DEADLINE RCLONE_CHECK_RETRY_DELAY; do
+    [[ ${!v} =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "$v must be a bounded integer"
+  done
+  (( RCLONE_CHECK_ATTEMPTS >= 1 && RCLONE_CHECK_ATTEMPTS <= 10 && RCLONE_CHECK_LOW_LEVEL_RETRIES >= 1 && RCLONE_CHECK_LOW_LEVEL_RETRIES <= 20 )) || die 'Invalid readback retry limits'
+  (( RCLONE_CHECK_TIMEOUT >= 1 && RCLONE_CHECK_TIMEOUT <= 3600 && RCLONE_CHECK_DEADLINE >= 1 && RCLONE_CHECK_DEADLINE <= 86400 && RCLONE_CHECK_RETRY_DELAY <= 3600 )) || die 'Invalid readback time limits'
   (( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
   (( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
   [[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
@@ -259,6 +267,7 @@ check_space() {
 
 check_remote() {
   need "$RCLONE_BIN"
+  need timeout
   [[ $RCLONE_REMOTE_BASE =~ ^[a-zA-Z0-9_-]+:[^:]+$ ]] || die "Use a named crypt remote with a dedicated non-empty path"
   [[ $RCLONE_REMOTE_BASE != *$'\n'* ]] || die "Invalid remote path"
   secure_file "$RCLONE_CONFIG"
@@ -278,6 +287,7 @@ check_remote() {
     case $key in
       RCLONE_BIN|RCLONE_CONFIG|RCLONE_CONFIG_PASS|RCLONE_REMOTE_BASE|RCLONE_RETRIES|\
       RCLONE_LOW_LEVEL_RETRIES|RCLONE_RETRIES_SLEEP|RCLONE_TIMEOUT|RCLONE_CONTIMEOUT|\
+      RCLONE_CHECK_ATTEMPTS|RCLONE_CHECK_RETRY_DELAY|RCLONE_CHECK_TIMEOUT|RCLONE_CHECK_DEADLINE|RCLONE_CHECK_LOW_LEVEL_RETRIES|\
       RCLONE_STATS_INTERVAL|RCLONE_TRANSFERS|RCLONE_CHECKERS|RCLONE_ONEDRIVE_CHUNK_SIZE|RCLONE_BWLIMIT) ;;
       *) die "Unsupported rclone environment override: $key; use documented app-backup settings and the remote config" ;;
     esac
@@ -298,6 +308,38 @@ check_remote() {
 # by callers. NOTICE makes periodic progress visible without verbose HTTP logs.
 rc() { "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@"; }
 
+remote_readback() {
+  local attempt code=1
+  # rclone check uses cmd.Run(false, ...): --retries does NOT repeat the whole
+  # check. A fresh process also refreshes backend/download state after failures.
+  # Always recheck every file; never turn a partial match report into success.
+  for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
+    echo "-- Remote readback attempt $attempt/$RCLONE_CHECK_ATTEMPTS (deadline ${RCLONE_CHECK_DEADLINE}s, idle ${RCLONE_CHECK_TIMEOUT}s)"
+    # GNU timeout bounds backend pacing/retries as well as stalled I/O. It owns
+    # a process group so TERM/KILL cannot leave an old check running on retry.
+    # Multiline stats expose active checks; byte totals alone can say 100% while
+    # a crypt header is still pending. Combined output names each check result.
+    if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
+        "$RCLONE_BIN" "${RCLONE_ARGS[@]}" check "$@" --download \
+        --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" \
+        --stats-one-line=false --stats-one-line-date=false --combined -; then
+      echo "-- Remote readback attempt $attempt passed (all selected files)"
+      return 0
+    else
+      code=$?
+    fi
+    echo "WARNING: Remote readback attempt $attempt/$RCLONE_CHECK_ATTEMPTS failed (exit $code; 124 means deadline exceeded)"
+    # Preserve cancellation semantics, including a forced kill after the grace
+    # period. Do not restart a command an operator/service manager has stopped.
+    if (( code >= 128 )); then return "$code"; fi
+    if (( attempt < RCLONE_CHECK_ATTEMPTS )); then
+      sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?
+    fi
+  done
+  echo 'ERROR: Remote readback exhausted; remote contents remain unverified'
+  return "$code"
+}
+
 # Exclusions are rooted so a nested directory named data is not accidentally lost.
 exclusions() {
   EXCLUDES=()
@@ -517,7 +559,15 @@ finish() {
   elif [[ ( $code == 0 || ( $OPERATION == restore && ${MODE:-} == verify ) ) && -n ${STAGING_DIR:-} ]]; then
     rm -rf -- "$STAGING_DIR" && STAGING_DIR='' || code=1
   fi
-  if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi
+  if [[ $code != 0 ]]; then
+    echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none} (temporary staging only)"
+    if [[ $OPERATION == backup && -n ${RUN_DIR:-} && -f $RUN_DIR/COMPLETE ]]; then
+      echo "Retained local backup: $RUN_DIR"
+      if [[ $ENABLE_UPLOAD == true && ! -f $RUN_DIR/UPLOADED ]]; then
+        echo 'Remote publication not confirmed; local backup retained for recovery.'
+      fi
+    fi
+  fi
   # Close the logger's writers and wait for EOF so the mail tail includes the
   # final recovery/status messages. Reporting itself uses the original stdout.
   exec 1>&"$REPORT_STDOUT" 2>&"$REPORT_STDERR"

lib/space.sh

@@ -21,7 +21,7 @@ ensure_space() {
       [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
       echo "-- Space recovery: verifying offsite copy before local removal: $name"
       if helper verify "$dir" >/dev/null &&
-         rc check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --download --exclude /UPLOADED; then
+         remote_readback "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED; then
         python3 "$LIB_DIR/space.py" remove "$dir"
         echo "-- Space recovery: removed local copy $name; verified offsite copy retained"
       else