Recover disk space safely and complete metadata-only backup updates

lib/common.sh

@@ -28,6 +28,7 @@ load_config() {
   : "${LOG_RETENTION_DAYS:=30}"
   : "${MIN_FREE_GB:=10}" "${LOCAL_RETENTION_DAYS:=7}" "${REMOTE_RETENTION_DAYS:=30}"
   : "${RETENTION_KEEP_LAST:=3}" "${ENABLE_REMOTE_RETENTION:=false}"
+  : "${SPACE_KEEP_LOCAL:=1}"
   : "${ENABLE_UPLOAD:=true}" "${RCLONE_BIN:=rclone}"
   : "${RCLONE_REMOTE_BASE:=}" "${RCLONE_CONFIG:=/root/.config/rclone/rclone.conf}"
   : "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}"
@@ -73,12 +74,13 @@ load_config() {
     [[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
   done
   (( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
+  [[ $SPACE_KEEP_LOCAL =~ ^[1-9][0-9]{0,5}$ ]] || die 'SPACE_KEEP_LOCAL must be an integer between 1 and 999999'
   # Seconds/counts only: never permit disabled deadlines or unbounded retries.
   for v in RCLONE_CHECK_ATTEMPTS RCLONE_CHECK_LOW_LEVEL_RETRIES RCLONE_CHECK_TIMEOUT RCLONE_CHECK_DEADLINE RCLONE_CHECK_RETRY_DELAY; do
     [[ ${!v} =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "$v must be a bounded integer"
   done
-  (( RCLONE_CHECK_ATTEMPTS >= 1 && RCLONE_CHECK_ATTEMPTS <= 10 && RCLONE_CHECK_LOW_LEVEL_RETRIES >= 1 && RCLONE_CHECK_LOW_LEVEL_RETRIES <= 20 )) || die 'Invalid readback retry limits'
-  (( RCLONE_CHECK_TIMEOUT >= 1 && RCLONE_CHECK_TIMEOUT <= 3600 && RCLONE_CHECK_DEADLINE >= 1 && RCLONE_CHECK_DEADLINE <= 86400 && RCLONE_CHECK_RETRY_DELAY <= 3600 )) || die 'Invalid readback time limits'
+  (( RCLONE_CHECK_ATTEMPTS >= 1 && RCLONE_CHECK_ATTEMPTS <= 10 && RCLONE_CHECK_LOW_LEVEL_RETRIES >= 1 && RCLONE_CHECK_LOW_LEVEL_RETRIES <= 20 )) || die 'Invalid remote check retry limits'
+  (( RCLONE_CHECK_TIMEOUT >= 1 && RCLONE_CHECK_TIMEOUT <= 3600 && RCLONE_CHECK_DEADLINE >= 1 && RCLONE_CHECK_DEADLINE <= 86400 && RCLONE_CHECK_RETRY_DELAY <= 3600 )) || die 'Invalid remote check time limits'
   (( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
   (( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
   [[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
@@ -277,8 +279,7 @@ check_remote() {
   for key in ${!RCLONE_CONFIG_@}; do
     [[ $key == RCLONE_CONFIG_PASS ]] || die "Remote overrides through $key are unsupported; use the config file"
   done
-  # rclone reads global flags from exported RCLONE_* variables too. For example,
-  # RCLONE_SIZE_ONLY bypasses byte comparison even with check --download; filters
+  # rclone reads global flags from exported RCLONE_* variables too. Filters
   # can hide missing archives from both copy and check. Refuse such ambient flags
   # before the first remote operation, rather than silently weakening verification.
   # Allow only our explicit settings (passed as CLI flags) and config decryption.
@@ -308,27 +309,26 @@ check_remote() {
 # by callers. NOTICE makes periodic progress visible without verbose HTTP logs.
 rc() { "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@"; }
 
-remote_readback() {
+remote_metadata_check() {
   local attempt code=1
   # rclone check uses cmd.Run(false, ...): --retries does NOT repeat the whole
-  # check. A fresh process also refreshes backend/download state after failures.
-  # Always recheck every file; never turn a partial match report into success.
+  # check. Retry the complete metadata comparison; never download archive data.
+  # Content integrity is checked by the monthly restore test.
   for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
-    echo "-- Remote readback attempt $attempt/$RCLONE_CHECK_ATTEMPTS (deadline ${RCLONE_CHECK_DEADLINE}s, idle ${RCLONE_CHECK_TIMEOUT}s)"
+    echo "-- Remote metadata check attempt $attempt/$RCLONE_CHECK_ATTEMPTS (deadline ${RCLONE_CHECK_DEADLINE}s, idle ${RCLONE_CHECK_TIMEOUT}s)"
     # GNU timeout bounds backend pacing/retries as well as stalled I/O. It owns
     # a process group so TERM/KILL cannot leave an old check running on retry.
-    # Multiline stats expose active checks; byte totals alone can say 100% while
-    # a crypt header is still pending. Combined output names each check result.
+    # Combined output names missing files and size mismatches.
     if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
-        "$RCLONE_BIN" "${RCLONE_ARGS[@]}" check "$@" --download \
+        "$RCLONE_BIN" "${RCLONE_ARGS[@]}" check "$@" --size-only \
         --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" \
         --stats-one-line=false --stats-one-line-date=false --combined -; then
-      echo "-- Remote readback attempt $attempt passed (all selected files)"
+      echo "-- Remote metadata check attempt $attempt passed (all selected files)"
       return 0
     else
       code=$?
     fi
-    echo "WARNING: Remote readback attempt $attempt/$RCLONE_CHECK_ATTEMPTS failed (exit $code; 124 means deadline exceeded)"
+    echo "WARNING: Remote metadata check attempt $attempt/$RCLONE_CHECK_ATTEMPTS failed (exit $code; 124 means deadline exceeded)"
     # Preserve cancellation semantics, including a forced kill after the grace
     # period. Do not restart a command an operator/service manager has stopped.
     if (( code >= 128 )); then return "$code"; fi
@@ -336,10 +336,39 @@ remote_readback() {
       sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?
     fi
   done
-  echo 'ERROR: Remote readback exhausted; remote contents remain unverified'
+  echo 'ERROR: Remote metadata check exhausted; remote file presence/sizes remain unverified'
   return "$code"
 }
 
+# Before reclaiming a local copy, bind the remote sizes to the exact local
+# manifest and completion marker. No archive download or temporary disk space.
+remote_identity_check() {
+  local dir=$1 remote=$2 file attempt code size
+  local -a statuses
+  need cmp
+  for file in manifest.json COMPLETE; do
+    size=$(stat -c %s -- "$dir/$file") || return 1
+    code=1
+    for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
+      if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
+          "$RCLONE_BIN" "${RCLONE_ARGS[@]}" cat "$remote/$file" --head "$((size + 1))" \
+          --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" |
+          cmp -s -- "$dir/$file" -; then
+        code=0
+        break
+      else
+        statuses=("${PIPESTATUS[@]}")
+        code=${statuses[0]}
+        if (( code == 0 )); then code=${statuses[1]}; fi
+      fi
+      (( code < 128 )) || return "$code"
+      echo "WARNING: Remote identity check failed: $file (attempt $attempt/$RCLONE_CHECK_ATTEMPTS)"
+      if (( attempt < RCLONE_CHECK_ATTEMPTS )); then sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?; fi
+    done
+    (( code == 0 )) || return "$code"
+  done
+}
+
 # Exclusions are rooted so a nested directory named data is not accidentally lost.
 exclusions() {
   EXCLUDES=()

lib/report.py

@@ -65,9 +65,9 @@ def deliver(args):
     with open(args.log, encoding='utf-8', errors='replace') as stream:
         tail = ''.join(deque(stream, maxlen=args.tail_lines))
     upload = {'NOT_STARTED': 'Nicht gestartet', 'DISABLED': 'Deaktiviert (nur lokal)',
-              'COPY': 'Nicht abgeschlossen: Upload', 'CHECK': 'Nicht abgeschlossen: Rückleseprüfung',
+              'COPY': 'Nicht abgeschlossen: Upload', 'CHECK': 'Nicht abgeschlossen: Dateiprüfung',
               'MARKER': 'Nicht abgeschlossen: Abschlussmarker',
-              'VERIFIED': 'OK (Inhalt und Abschlussmarker zurückgelesen)'}[args.upload_status]
+              'VERIFIED': 'OK (Dateien vorhanden, Größen und Abschlussmarker geprüft; Inhaltsprüfung beim Restore-Test)'}[args.upload_status]
     body = [
         'Backup Report', '=============', f'Zeit: {now.isoformat(timespec="seconds")}',
         f'Host: {host}', f'Status: {state}', f'Exit code: {args.exit_code}',

lib/space.py

@@ -9,6 +9,7 @@ import os
 from pathlib import Path
 import re
 import shutil
+import subprocess
 
 import backup_set
 
@@ -18,12 +19,21 @@ def private(path):
             path.stat().st_uid == 0 and not path.stat().st_mode & 0o077)
 
 
+def mounted_paths():
+    # st_dev/ismount alone miss same-filesystem bind mounts. Linux mountinfo
+    # lists file and directory mounts, escaping whitespace and backslashes.
+    return {Path(re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)), line.split()[4]))
+            for line in Path('/proc/self/mountinfo').read_text().splitlines()}
+
+
 def remove_tree(path):
     if not private(path):
         raise ValueError('Refusing non-private/aliased scratch: ' + str(path))
+    if os.path.ismount(path) or any(p == path or path in p.parents for p in mounted_paths()):
+        raise ValueError('Refusing mounted scratch: ' + str(path))
     device = path.stat().st_dev
-    for parent, dirs, _ in os.walk(path, followlinks=False):
-        for name in dirs:
+    for parent, dirs, files in os.walk(path, followlinks=False):
+        for name in dirs + files:
             child = Path(parent) / name
             if not child.is_symlink() and (child.stat().st_dev != device or os.path.ismount(child)):
                 raise ValueError('Refusing mounted scratch: ' + str(child))
@@ -66,7 +76,9 @@ def clean(work, staging, archives, prefix, restore=None):
                 print('-- Removed abandoned verification download: ' + path.name, flush=True)
 
 
-def eligible(archives, prefix, keep):
+def uploaded_sets(archives, prefix):
+    if not private(archives):
+        raise ValueError('Cleanup requires private canonical archives: ' + str(archives))
     valid = []
     for path in archives.iterdir():
         if not private(path) or not path.name.startswith(prefix+'_'):
@@ -81,19 +93,48 @@ def eligible(archives, prefix, keep):
         except (ValueError, OSError, KeyError, TypeError):
             continue
         valid.append(path.name)
-    # Emergency pruning may ignore age, NEVER the minimum newest-copy count.
-    for name in sorted(valid, reverse=True)[keep:][::-1]:
+    return sorted(valid, reverse=True)
+
+
+def eligible(archives, prefix, keep, protected=False):
+    if keep < 1:
+        raise ValueError('At least one uploaded local set must be retained')
+    # Pressure policy is independent of normal age/count retention.
+    valid = uploaded_sets(archives, prefix)
+    for name in (valid[:keep] if protected else valid[keep:][::-1]):
         print(name)
 
 
+def inventory(archives, prefix, keep):
+    valid = uploaded_sets(archives, prefix)
+    protected = set(valid[:keep])
+    print('-- Remaining local archive inventory (allocated bytes; no deletion):', flush=True)
+    for path in sorted(archives.iterdir()):
+        if not private(path):
+            print(f'   {path.name}: protected unknown/aliased/non-private entry', flush=True)
+            continue
+        if path.name in protected:
+            reason = 'protected local minimum'
+        elif path.name in valid:
+            reason = 'uploaded; retained (verification/source/device constraints)'
+        else:
+            reason = 'protected unconfirmed/incomplete/unknown set'
+        result = subprocess.run(['du', '-sx', '-B1', '--', str(path)],
+                                text=True, capture_output=True, check=True)
+        size = int(result.stdout.split()[0])
+        print(f'   {path.name}: {size} bytes; {reason}', flush=True)
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('mode', choices=('clean', 'eligible', 'remove'))
+    parser.add_argument('mode', choices=('clean', 'eligible', 'protected', 'inventory', 'remove'))
     parser.add_argument('paths', nargs='+')
     args = parser.parse_args()
     if args.mode == 'clean':
         clean(*(Path(p) for p in args.paths[:3]), args.paths[3], Path(args.paths[4]) if len(args.paths) > 4 else None)
-    elif args.mode == 'eligible':
-        eligible(Path(args.paths[0]), args.paths[1], int(args.paths[2]))
+    elif args.mode in ('eligible', 'protected'):
+        eligible(Path(args.paths[0]), args.paths[1], int(args.paths[2]), args.mode == 'protected')
+    elif args.mode == 'inventory':
+        inventory(Path(args.paths[0]), args.paths[1], int(args.paths[2]))
     else:
         remove_tree(Path(args.paths[0]))

lib/space.sh

@@ -4,34 +4,66 @@ clean_backup_scratch() {
   python3 "$LIB_DIR/space.py" clean "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$RESTORE_ROOT"
 }
 
+available_space() {
+  local value
+  value=$(df -PB1 -- "$1" | awk 'END{print $4}') || return 1
+  [[ $value =~ ^(0|[1-9][0-9]{0,17})$ ]] || die "Cannot determine available space on $1"
+  printf '%s\n' "$value"
+}
+
 ensure_space() {
-  local path=$1 additional=${2:-0} avail name dir candidates
+  local path=$1 additional=${2:-0} avail name dir candidates code protected
   [[ $additional =~ ^(0|[1-9][0-9]{0,15})$ ]] || die 'Invalid or excessive space request'
   local required=$((MIN_FREE_GB * 1024 * 1024 * 1024 + additional))
-  avail=$(df -PB1 -- "$path" | awk 'END{print $4}')
+  avail=$(available_space "$path")
   (( avail < required )) || return 0
   echo "-- Space recovery: need $required bytes, available $avail on $path"
+  echo "-- Space policy: keep $SPACE_KEEP_LOCAL uploaded local set(s) under pressure; normal retention keeps $RETENTION_KEEP_LAST"
   if [[ $ENABLE_UPLOAD == true && $(stat -c %d "$path") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
     check_remote
-    candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$RETENTION_KEEP_LAST")
+    candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
+    if [[ -n $candidates ]]; then
+      protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
+      [[ -n $protected ]] || die 'No protected local backup; refusing space recovery'
+      while IFS= read -r name; do
+        echo "-- Space recovery: validating protected local copy: $name"
+        if helper verify "$ARCHIVE_DIR/$name" >/dev/null; then continue; else code=$?; fi
+        (( code < 128 )) || return "$code"
+        echo "WARNING: Protected local copy is damaged; keeping all candidates: $name"
+        candidates=''
+        break
+      done <<< "$protected"
+    fi
     while IFS= read -r name; do
       [[ -n $name ]] || continue
+      avail=$(available_space "$path")
+      (( avail < required )) || return 0
       dir="$ARCHIVE_DIR/$name"
       # A manually selected local restore source is never a cleanup candidate.
       [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
       echo "-- Space recovery: verifying offsite copy before local removal: $name"
       if helper verify "$dir" >/dev/null &&
-         remote_readback "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED; then
+         remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
+         remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
         python3 "$LIB_DIR/space.py" remove "$dir"
-        echo "-- Space recovery: removed local copy $name; verified offsite copy retained"
+        echo "-- Space recovery: removed local copy $name; offsite copy with matching file sizes retained"
       else
+        code=$?
+        (( code < 128 )) || return "$code"
         echo "WARNING: Keeping $name: local/offsite verification failed"
       fi
-      avail=$(df -PB1 -- "$path" | awk 'END{print $4}')
-      (( avail < required )) || return 0
+      avail=$(available_space "$path")
+      if (( avail >= required )); then
+        echo "-- Space recovery complete: available $avail bytes, required $required"
+        return 0
+      fi
     done <<< "$candidates"
   fi
-  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail; protected backups/recovery data retained"
+  avail=$(available_space "$path")
+  (( avail < required )) || return 0
+  python3 "$LIB_DIR/space.py" inventory "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL" ||
+    echo 'WARNING: Local archive inventory incomplete; no further cleanup attempted'
+  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: resolve pending uploads or move backup workspace to a larger filesystem."
 }
 
 prepare_backup_space() {