Recover backup capacity with verified adaptive retention

Resume pending publications under local pressure, allow verified offsite-only history as a final local recovery stage, and prune old complete remote sets against actual upload quota while preserving the newest verified copies. Document defaults, recovery limits and validation; add pressure and quota regression coverage.

lib/common.sh

@@ -28,7 +28,8 @@ load_config() {
   : "${LOG_RETENTION_DAYS:=30}"
   : "${MIN_FREE_GB:=10}" "${LOCAL_RETENTION_DAYS:=7}" "${REMOTE_RETENTION_DAYS:=30}"
   : "${RETENTION_KEEP_LAST:=3}" "${ENABLE_REMOTE_RETENTION:=false}"
-  : "${SPACE_KEEP_LOCAL:=1}"
+  : "${SPACE_KEEP_LOCAL:=1}" "${SPACE_ALLOW_REMOTE_ONLY:=true}"
+  : "${SPACE_REMOTE_CLEANUP:=true}" "${SPACE_KEEP_REMOTE:=1}"
   : "${ENABLE_UPLOAD:=true}" "${RCLONE_BIN:=rclone}"
   : "${RCLONE_REMOTE_BASE:=}" "${RCLONE_CONFIG:=/root/.config/rclone/rclone.conf}"
   : "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}"
@@ -67,7 +68,7 @@ load_config() {
   fi
   for v in ENABLE_UPLOAD ENABLE_REMOTE_RETENTION ENABLE_MAIL_REPORT ENABLE_WORDPRESS ENABLE_NEXTCLOUD \
       ENABLE_NEXTCLOUD_DATA ENABLE_GITEA ENABLE_MAIL ENABLE_MAIL_STORE ENABLE_DB_DUMPS ENABLE_NEXTCLOUD_MAINTENANCE \
-      ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE ENABLE_RESTORE_TEST; do
+      ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE ENABLE_RESTORE_TEST SPACE_ALLOW_REMOTE_ONLY SPACE_REMOTE_CLEANUP; do
     [[ ${!v} == true || ${!v} == false ]] || die "$v must be true or false"
   done
   for v in LOG_RETENTION_DAYS MIN_FREE_GB LOCAL_RETENTION_DAYS REMOTE_RETENTION_DAYS RETENTION_KEEP_LAST ZSTD_LEVEL ZSTD_THREADS; do
@@ -75,6 +76,7 @@ load_config() {
   done
   (( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
   [[ $SPACE_KEEP_LOCAL =~ ^[1-9][0-9]{0,5}$ ]] || die 'SPACE_KEEP_LOCAL must be an integer between 1 and 999999'
+  [[ $SPACE_KEEP_REMOTE =~ ^[1-9][0-9]{0,5}$ ]] || die 'SPACE_KEEP_REMOTE must be an integer between 1 and 999999'
   # Seconds/counts only: never permit disabled deadlines or unbounded retries.
   for v in RCLONE_CHECK_ATTEMPTS RCLONE_CHECK_LOW_LEVEL_RETRIES RCLONE_CHECK_TIMEOUT RCLONE_CHECK_DEADLINE RCLONE_CHECK_RETRY_DELAY; do
     [[ ${!v} =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "$v must be a bounded integer"

lib/space.py

@@ -5,7 +5,9 @@ Only private, recognized backup scratch is disposable. Recovery journals and
 manual restore extractions always win over attempts to reclaim disk space.
 """
 import argparse
+import json
 import os
+import sys
 from pathlib import Path
 import re
 import shutil
@@ -105,6 +107,48 @@ def eligible(archives, prefix, keep, protected=False):
         print(name)
 
 
+def pending(archives, prefix):
+    if not private(archives):
+        raise ValueError('Pending uploads require private canonical archives')
+    for path in sorted(archives.iterdir()):
+        if not private(path) or not path.name.startswith(prefix+'_') or (path/'UPLOADED').exists() or (path/'UPLOADED').is_symlink():
+            continue
+        try:
+            data = backup_set.load_set(path, metadata_only=True)
+            if data['run_id'] == path.name:
+                print(path.name)
+        except (ValueError, OSError, KeyError, TypeError):
+            continue
+
+
+def remote_valid(folder, name):
+    data = backup_set.load_set(folder, metadata_only=True)
+    if data['run_id'] != name:
+        raise ValueError('Remote directory/manifest identity mismatch')
+    entries = json.loads((folder/'listing.json').read_text())
+    expected = {a['file']: a['size'] for a in data['archives']}
+    expected.update({p: (folder/p).stat().st_size for p in ('manifest.json', 'COMPLETE')})
+    actual = {}
+    for entry in entries:
+        path = entry['Path']
+        if path in actual or entry['IsDir'] or type(entry['Size']) is not int:
+            raise ValueError('Invalid remote file inventory')
+        actual[path] = entry['Size']
+    if actual != expected:
+        raise ValueError('Remote files missing, unexpected or wrong sizes')
+
+
+def quota():
+    try:
+        data = json.load(sys.stdin)
+    except (ValueError, TypeError):
+        sys.exit(1)
+    free = data.get('free') if isinstance(data, dict) else None
+    if type(free) is not int or not 0 <= free <= 10**18:
+        sys.exit(1)
+    print(free)
+
+
 def inventory(archives, prefix, keep):
     valid = uploaded_sets(archives, prefix)
     protected = set(valid[:keep])
@@ -127,10 +171,16 @@ def inventory(archives, prefix, keep):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('mode', choices=('clean', 'eligible', 'protected', 'inventory', 'remove'))
-    parser.add_argument('paths', nargs='+')
+    parser.add_argument('mode', choices=('clean', 'eligible', 'protected', 'inventory', 'remove', 'pending', 'remote-valid', 'quota'))
+    parser.add_argument('paths', nargs='*')
     args = parser.parse_args()
-    if args.mode == 'clean':
+    if args.mode == 'quota':
+        quota()
+    elif args.mode == 'remote-valid':
+        remote_valid(Path(args.paths[0]), args.paths[1])
+    elif args.mode == 'pending':
+        pending(Path(args.paths[0]), args.paths[1])
+    elif args.mode == 'clean':
         clean(*(Path(p) for p in args.paths[:3]), args.paths[3], Path(args.paths[4]) if len(args.paths) > 4 else None)
     elif args.mode in ('eligible', 'protected'):
         eligible(Path(args.paths[0]), args.paths[1], int(args.paths[2]), args.mode == 'protected')

lib/space.sh

@@ -12,7 +12,7 @@ available_space() {
 }
 
 ensure_space() {
-  local path=$1 additional=${2:-0} avail name dir candidates code protected
+  local path=$1 additional=${2:-0} avail name dir candidates code protected=''
   [[ $additional =~ ^(0|[1-9][0-9]{0,15})$ ]] || die 'Invalid or excessive space request'
   local required=$((MIN_FREE_GB * 1024 * 1024 * 1024 + additional))
   avail=$(available_space "$path")
@@ -21,16 +21,17 @@ ensure_space() {
   echo "-- Space policy: keep $SPACE_KEEP_LOCAL uploaded local set(s) under pressure; normal retention keeps $RETENTION_KEEP_LAST"
   if [[ $ENABLE_UPLOAD == true && $(stat -c %d "$path") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
     check_remote
+    resume_pending_uploads
     candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
-    if [[ -n $candidates ]]; then
-      protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
-      [[ -n $protected ]] || die 'No protected local backup; refusing space recovery'
+    protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
+    if [[ -n $protected ]]; then
       while IFS= read -r name; do
         echo "-- Space recovery: validating protected local copy: $name"
         if helper verify "$ARCHIVE_DIR/$name" >/dev/null; then continue; else code=$?; fi
         (( code < 128 )) || return "$code"
         echo "WARNING: Protected local copy is damaged; keeping all candidates: $name"
         candidates=''
+        protected=''
         break
       done <<< "$protected"
     fi
@@ -58,12 +59,13 @@ ensure_space() {
         return 0
       fi
     done <<< "$candidates"
+    release_local_floor "$path" "$required" "$protected"
   fi
   avail=$(available_space "$path")
   (( avail < required )) || return 0
   python3 "$LIB_DIR/space.py" inventory "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL" ||
     echo 'WARNING: Local archive inventory incomplete; no further cleanup attempted'
-  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: resolve pending uploads or move backup workspace to a larger filesystem."
+  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: remote verification/quota failed, protected data or workspace capacity prevents further safe cleanup."
 }
 
 prepare_backup_space() {
@@ -100,3 +102,178 @@ prepare_backup_space() {
     ensure_space "$ARCHIVE_DIR" "$total"
   fi
 }
+
+# Bound listing/quota/control-file operations just like remote verification.
+space_rc() {
+  local attempt code=1
+  for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
+    if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
+        "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@" \
+        --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES"; then
+      return 0
+    else code=$?; fi
+    (( code < 128 )) || return "$code"
+    if (( attempt < RCLONE_CHECK_ATTEMPTS )); then sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?; fi
+  done
+  return "$code"
+}
+
+remote_free_space() {
+  local -a statuses
+  if space_rc about "$RCLONE_REMOTE_BASE" --json | python3 "$LIB_DIR/space.py" quota; then return 0; else
+    statuses=("${PIPESTATUS[@]}")
+    if (( statuses[0] != 0 )); then return "${statuses[0]}"; fi
+    return "${statuses[1]}"
+  fi
+}
+
+# Read only small control files and the decrypted size inventory. Identity must
+# match the directory, and every archive must exist with its manifest size.
+space_remote_valid() {
+  local name=$1 folder=$2 remote="${RCLONE_REMOTE_BASE%/}/$1"
+  space_rc cat "$remote/COMPLETE" --head 256 > "$folder/COMPLETE" &&
+    space_rc cat "$remote/manifest.json" --head 1048577 > "$folder/manifest.json" &&
+    space_rc lsjson "$remote" --max-depth 1 > "$folder/listing.json" &&
+    python3 "$LIB_DIR/space.py" remote-valid "$folder" "$name"
+}
+
+ensure_remote_space() (
+  local additional=$1 exclude=${2:-} free scratch name anchor code kept=0
+  additional=$((additional + additional / 20 + 1048576))
+  [[ $ENABLE_UPLOAD == true && $SPACE_REMOTE_CLEANUP == true ]] || return 0
+  if free=$(remote_free_space); then :; else
+    code=$?
+    (( code < 128 )) || return "$code"
+    echo 'WARNING: Remote quota unavailable; no automatic remote pressure cleanup (normal upload will still be attempted)'
+    return 0
+  fi
+  (( free < additional )) || return 0
+  echo "-- Remote space recovery: need $additional bytes, available $free; keep $SPACE_KEEP_REMOTE complete offsite set(s)"
+  scratch=$(mktemp -d "$STAGING_ROOT/.remote-space.XXXXXX") || return 1
+  trap 'rm -rf -- "$scratch"' EXIT
+  trap 'exit 130' INT
+  trap 'exit 143' TERM
+  if space_rc lsf "$RCLONE_REMOTE_BASE" --dirs-only --max-depth 1 > "$scratch/dirs"; then :; else
+    code=$?
+    # No deletion on an incomplete listing. Keep cancellation semantics.
+    rm -rf -- "$scratch"
+    return "$code"
+  fi
+  : > "$scratch/valid" || return 1
+  # Newest first: quota cleanup always protects the newest verified offsite sets.
+  sort -r "$scratch/dirs" > "$scratch/sorted" || return 1
+  while IFS= read -r name; do
+    name=${name%/}
+    [[ $name =~ ^${ARCHIVE_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}_[0-9a-f]{8}$ ]] || continue
+    [[ $name != "$exclude" && $name != "${REMOTE_RUN:-}" ]] || continue
+    if space_remote_valid "$name" "$scratch"; then
+      printf '%s\n' "$name" >> "$scratch/valid" || return 1
+    else
+      code=$?
+      if (( code >= 128 )); then rm -rf -- "$scratch"; return "$code"; fi
+      echo "WARNING: Remote pressure cleanup preserves unverified/incomplete set: $name"
+    fi
+  done < "$scratch/sorted"
+  : > "$scratch/keep" || return 1
+  : > "$scratch/candidates" || return 1
+  while IFS= read -r name; do
+    if (( kept < SPACE_KEEP_REMOTE )); then
+      echo "-- Remote pressure cleanup protects: $name"
+      printf '%s\n' "$name" >> "$scratch/keep" || return 1
+      kept=$((kept + 1))
+    else printf '%s\n' "$name" >> "$scratch/candidates" || return 1; fi
+  done < "$scratch/valid"
+  sort "$scratch/candidates" > "$scratch/delete" || return 1
+  while IFS= read -r name; do
+    while IFS= read -r anchor; do
+      if space_remote_valid "$anchor" "$scratch"; then :; else
+        code=$?; rm -rf -- "$scratch"; return "$code"
+      fi
+    done < "$scratch/keep"
+    if space_remote_valid "$name" "$scratch"; then :; else
+      code=$?; rm -rf -- "$scratch"; return "$code"
+    fi
+    echo "-- Remote pressure cleanup: removing old complete set $name"
+    if space_rc purge "${RCLONE_REMOTE_BASE%/}/$name"; then :; else
+      code=$?; rm -rf -- "$scratch"; return "$code"
+    fi
+    # Provider quota/recycle-bin accounting can lag: never assume purge freed it.
+    if free=$(remote_free_space); then :; else
+      code=$?; rm -rf -- "$scratch"; return "$code"
+    fi
+    if (( free >= additional )); then rm -rf -- "$scratch"; return 0; fi
+  done < "$scratch/delete"
+  rm -rf -- "$scratch"
+  echo "ERROR: Insufficient remote quota after safe cleanup: need $additional bytes, available $free; newest offsite backups retained"
+  return 1
+)
+
+resume_pending_uploads() {
+  local name dir code size
+  local pending
+  pending=$(python3 "$LIB_DIR/space.py" pending "$ARCHIVE_DIR" "$ARCHIVE_PREFIX") || return 1
+  while IFS= read -r name; do
+    [[ -n $name ]] || continue
+    dir="$ARCHIVE_DIR/$name"
+    [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+    echo "-- Space recovery: retrying pending publication: $name"
+    if helper verify "$dir" >/dev/null; then :; else
+      code=$?; (( code < 128 )) || return "$code"
+      echo "WARNING: Keeping invalid pending local set: $name"; continue
+    fi
+    if remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
+       remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
+      date -u -Is > "$dir/UPLOADED" || return 1
+      sync -f "$dir" || return 1
+      echo "-- Pending publication already complete offsite: $name"
+      continue
+    else
+      code=$?; (( code < 128 )) || return "$code"
+    fi
+    size=$(du -sb -- "$dir" | awk '{print $1}') || return 1
+    if ensure_remote_space "$size" "$name" &&
+       rc copy "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /COMPLETE --exclude /UPLOADED --immutable &&
+       remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /COMPLETE --exclude /UPLOADED &&
+       rc copyto "$dir/COMPLETE" "${RCLONE_REMOTE_BASE%/}/$name/COMPLETE" --immutable &&
+       remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
+      date -u -Is > "$dir/UPLOADED" || return 1
+      sync -f "$dir" || return 1
+      echo "-- Pending publication confirmed: $name"
+    else
+      code=$?; (( code < 128 )) || return "$code"
+      echo "WARNING: Pending upload remains protected: $name"
+    fi
+  done <<< "$pending"
+}
+
+release_local_floor() {
+  local path=$1 required=$2 protected=$3 name dir code bytes available reclaimable=0
+  [[ $SPACE_ALLOW_REMOTE_ONLY == true && -n $protected ]] || return 0
+  # Do not sacrifice the local floor if its allocated blocks cannot solve the
+  # deficit. This also avoids deleting the last local set on undersized disks.
+  available=$(available_space "$path") || return 1
+  while IFS= read -r name; do
+    dir="$ARCHIVE_DIR/$name"
+    [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+    bytes=$(du -sx -B1 -- "$dir" | awk '{print $1}') || return 1
+    reclaimable=$((reclaimable + bytes))
+  done <<< "$protected"
+  (( available + reclaimable >= required )) || return 0
+  echo '-- Emergency space recovery: allowing offsite-only history until the new backup completes'
+  # protected is newest first. Release oldest first, stopping at the target.
+  while IFS= read -r name; do
+    dir="$ARCHIVE_DIR/$name"
+    [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+    # ensure_space already validated all protected local contents under the lock.
+    if remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
+       remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
+      python3 "$LIB_DIR/space.py" remove "$dir" || return 1
+      echo "-- Emergency space recovery: removed local copy $name; verified offsite set retained"
+    else
+      code=$?; (( code < 128 )) || return "$code"
+      echo "WARNING: Emergency cleanup keeps $name: local/offsite verification failed"
+    fi
+    available=$(available_space "$path") || return 1
+    (( available < required )) || return 0
+  done < <(sort <<< "$protected")
+}