Recover backup capacity with verified adaptive retention

Resume pending publications under local pressure, allow verified offsite-only history as a final local recovery stage, and prune old complete remote sets against actual upload quota while preserving the newest verified copies. Document defaults, recovery limits and validation; add pressure and quota regression coverage.

lib/space.sh

@@ -12,7 +12,7 @@ available_space() {
 }
 
 ensure_space() {
-  local path=$1 additional=${2:-0} avail name dir candidates code protected
+  local path=$1 additional=${2:-0} avail name dir candidates code protected=''
   [[ $additional =~ ^(0|[1-9][0-9]{0,15})$ ]] || die 'Invalid or excessive space request'
   local required=$((MIN_FREE_GB * 1024 * 1024 * 1024 + additional))
   avail=$(available_space "$path")
@@ -21,16 +21,17 @@ ensure_space() {
   echo "-- Space policy: keep $SPACE_KEEP_LOCAL uploaded local set(s) under pressure; normal retention keeps $RETENTION_KEEP_LAST"
   if [[ $ENABLE_UPLOAD == true && $(stat -c %d "$path") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
     check_remote
+    resume_pending_uploads
     candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
-    if [[ -n $candidates ]]; then
-      protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
-      [[ -n $protected ]] || die 'No protected local backup; refusing space recovery'
+    protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
+    if [[ -n $protected ]]; then
       while IFS= read -r name; do
         echo "-- Space recovery: validating protected local copy: $name"
         if helper verify "$ARCHIVE_DIR/$name" >/dev/null; then continue; else code=$?; fi
         (( code < 128 )) || return "$code"
         echo "WARNING: Protected local copy is damaged; keeping all candidates: $name"
         candidates=''
+        protected=''
         break
       done <<< "$protected"
     fi
@@ -58,12 +59,13 @@ ensure_space() {
         return 0
       fi
     done <<< "$candidates"
+    release_local_floor "$path" "$required" "$protected"
   fi
   avail=$(available_space "$path")
   (( avail < required )) || return 0
   python3 "$LIB_DIR/space.py" inventory "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL" ||
     echo 'WARNING: Local archive inventory incomplete; no further cleanup attempted'
-  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: resolve pending uploads or move backup workspace to a larger filesystem."
+  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: remote verification/quota failed, protected data or workspace capacity prevents further safe cleanup."
 }
 
 prepare_backup_space() {
@@ -100,3 +102,178 @@ prepare_backup_space() {
     ensure_space "$ARCHIVE_DIR" "$total"
   fi
 }
+
+# Bound listing/quota/control-file operations just like remote verification.
+space_rc() {
+  local attempt code=1
+  for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
+    if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
+        "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@" \
+        --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES"; then
+      return 0
+    else code=$?; fi
+    (( code < 128 )) || return "$code"
+    if (( attempt < RCLONE_CHECK_ATTEMPTS )); then sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?; fi
+  done
+  return "$code"
+}
+
+remote_free_space() {
+  local -a statuses
+  if space_rc about "$RCLONE_REMOTE_BASE" --json | python3 "$LIB_DIR/space.py" quota; then return 0; else
+    statuses=("${PIPESTATUS[@]}")
+    if (( statuses[0] != 0 )); then return "${statuses[0]}"; fi
+    return "${statuses[1]}"
+  fi
+}
+
+# Read only small control files and the decrypted size inventory. Identity must
+# match the directory, and every archive must exist with its manifest size.
+space_remote_valid() {
+  local name=$1 folder=$2 remote="${RCLONE_REMOTE_BASE%/}/$1"
+  space_rc cat "$remote/COMPLETE" --head 256 > "$folder/COMPLETE" &&
+    space_rc cat "$remote/manifest.json" --head 1048577 > "$folder/manifest.json" &&
+    space_rc lsjson "$remote" --max-depth 1 > "$folder/listing.json" &&
+    python3 "$LIB_DIR/space.py" remote-valid "$folder" "$name"
+}
+
+ensure_remote_space() (
+  local additional=$1 exclude=${2:-} free scratch name anchor code kept=0
+  additional=$((additional + additional / 20 + 1048576))
+  [[ $ENABLE_UPLOAD == true && $SPACE_REMOTE_CLEANUP == true ]] || return 0
+  if free=$(remote_free_space); then :; else
+    code=$?
+    (( code < 128 )) || return "$code"
+    echo 'WARNING: Remote quota unavailable; no automatic remote pressure cleanup (normal upload will still be attempted)'
+    return 0
+  fi
+  (( free < additional )) || return 0
+  echo "-- Remote space recovery: need $additional bytes, available $free; keep $SPACE_KEEP_REMOTE complete offsite set(s)"
+  scratch=$(mktemp -d "$STAGING_ROOT/.remote-space.XXXXXX") || return 1
+  trap 'rm -rf -- "$scratch"' EXIT
+  trap 'exit 130' INT
+  trap 'exit 143' TERM
+  if space_rc lsf "$RCLONE_REMOTE_BASE" --dirs-only --max-depth 1 > "$scratch/dirs"; then :; else
+    code=$?
+    # No deletion on an incomplete listing. Keep cancellation semantics.
+    rm -rf -- "$scratch"
+    return "$code"
+  fi
+  : > "$scratch/valid" || return 1
+  # Newest first: quota cleanup always protects the newest verified offsite sets.
+  sort -r "$scratch/dirs" > "$scratch/sorted" || return 1
+  while IFS= read -r name; do
+    name=${name%/}
+    [[ $name =~ ^${ARCHIVE_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}_[0-9a-f]{8}$ ]] || continue
+    [[ $name != "$exclude" && $name != "${REMOTE_RUN:-}" ]] || continue
+    if space_remote_valid "$name" "$scratch"; then
+      printf '%s\n' "$name" >> "$scratch/valid" || return 1
+    else
+      code=$?
+      if (( code >= 128 )); then rm -rf -- "$scratch"; return "$code"; fi
+      echo "WARNING: Remote pressure cleanup preserves unverified/incomplete set: $name"
+    fi
+  done < "$scratch/sorted"
+  : > "$scratch/keep" || return 1
+  : > "$scratch/candidates" || return 1
+  while IFS= read -r name; do
+    if (( kept < SPACE_KEEP_REMOTE )); then
+      echo "-- Remote pressure cleanup protects: $name"
+      printf '%s\n' "$name" >> "$scratch/keep" || return 1
+      kept=$((kept + 1))
+    else printf '%s\n' "$name" >> "$scratch/candidates" || return 1; fi
+  done < "$scratch/valid"
+  sort "$scratch/candidates" > "$scratch/delete" || return 1
+  while IFS= read -r name; do
+    while IFS= read -r anchor; do
+      if space_remote_valid "$anchor" "$scratch"; then :; else
+        code=$?; rm -rf -- "$scratch"; return "$code"
+      fi
+    done < "$scratch/keep"
+    if space_remote_valid "$name" "$scratch"; then :; else
+      code=$?; rm -rf -- "$scratch"; return "$code"
+    fi
+    echo "-- Remote pressure cleanup: removing old complete set $name"
+    if space_rc purge "${RCLONE_REMOTE_BASE%/}/$name"; then :; else
+      code=$?; rm -rf -- "$scratch"; return "$code"
+    fi
+    # Provider quota/recycle-bin accounting can lag: never assume purge freed it.
+    if free=$(remote_free_space); then :; else
+      code=$?; rm -rf -- "$scratch"; return "$code"
+    fi
+    if (( free >= additional )); then rm -rf -- "$scratch"; return 0; fi
+  done < "$scratch/delete"
+  rm -rf -- "$scratch"
+  echo "ERROR: Insufficient remote quota after safe cleanup: need $additional bytes, available $free; newest offsite backups retained"
+  return 1
+)
+
+resume_pending_uploads() {
+  local name dir code size
+  local pending
+  pending=$(python3 "$LIB_DIR/space.py" pending "$ARCHIVE_DIR" "$ARCHIVE_PREFIX") || return 1
+  while IFS= read -r name; do
+    [[ -n $name ]] || continue
+    dir="$ARCHIVE_DIR/$name"
+    [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+    echo "-- Space recovery: retrying pending publication: $name"
+    if helper verify "$dir" >/dev/null; then :; else
+      code=$?; (( code < 128 )) || return "$code"
+      echo "WARNING: Keeping invalid pending local set: $name"; continue
+    fi
+    if remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
+       remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
+      date -u -Is > "$dir/UPLOADED" || return 1
+      sync -f "$dir" || return 1
+      echo "-- Pending publication already complete offsite: $name"
+      continue
+    else
+      code=$?; (( code < 128 )) || return "$code"
+    fi
+    size=$(du -sb -- "$dir" | awk '{print $1}') || return 1
+    if ensure_remote_space "$size" "$name" &&
+       rc copy "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /COMPLETE --exclude /UPLOADED --immutable &&
+       remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /COMPLETE --exclude /UPLOADED &&
+       rc copyto "$dir/COMPLETE" "${RCLONE_REMOTE_BASE%/}/$name/COMPLETE" --immutable &&
+       remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
+      date -u -Is > "$dir/UPLOADED" || return 1
+      sync -f "$dir" || return 1
+      echo "-- Pending publication confirmed: $name"
+    else
+      code=$?; (( code < 128 )) || return "$code"
+      echo "WARNING: Pending upload remains protected: $name"
+    fi
+  done <<< "$pending"
+}
+
+release_local_floor() {
+  local path=$1 required=$2 protected=$3 name dir code bytes available reclaimable=0
+  [[ $SPACE_ALLOW_REMOTE_ONLY == true && -n $protected ]] || return 0
+  # Do not sacrifice the local floor if its allocated blocks cannot solve the
+  # deficit. This also avoids deleting the last local set on undersized disks.
+  available=$(available_space "$path") || return 1
+  while IFS= read -r name; do
+    dir="$ARCHIVE_DIR/$name"
+    [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+    bytes=$(du -sx -B1 -- "$dir" | awk '{print $1}') || return 1
+    reclaimable=$((reclaimable + bytes))
+  done <<< "$protected"
+  (( available + reclaimable >= required )) || return 0
+  echo '-- Emergency space recovery: allowing offsite-only history until the new backup completes'
+  # protected is newest first. Release oldest first, stopping at the target.
+  while IFS= read -r name; do
+    dir="$ARCHIVE_DIR/$name"
+    [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+    # ensure_space already validated all protected local contents under the lock.
+    if remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
+       remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
+      python3 "$LIB_DIR/space.py" remove "$dir" || return 1
+      echo "-- Emergency space recovery: removed local copy $name; verified offsite set retained"
+    else
+      code=$?; (( code < 128 )) || return "$code"
+      echo "WARNING: Emergency cleanup keeps $name: local/offsite verification failed"
+    fi
+    available=$(available_space "$path") || return 1
+    (( available < required )) || return 0
+  done < <(sort <<< "$protected")
+}