admin
6737405b35
Resume pending publications under local pressure, allow verified offsite-only history as a final local recovery stage, and prune old complete remote sets against actual upload quota while preserving the newest verified copies. Document defaults, recovery limits and validation; add pressure and quota regression coverage.
280 lines
13 KiB
Bash
280 lines
13 KiB
Bash
#!/usr/bin/env bash
|
|
# Called only under the shared lock; --check never invokes this maintenance.
|
|
clean_backup_scratch() {
|
|
python3 "$LIB_DIR/space.py" clean "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$RESTORE_ROOT"
|
|
}
|
|
|
|
available_space() {
|
|
local value
|
|
value=$(df -PB1 -- "$1" | awk 'END{print $4}') || return 1
|
|
[[ $value =~ ^(0|[1-9][0-9]{0,17})$ ]] || die "Cannot determine available space on $1"
|
|
printf '%s\n' "$value"
|
|
}
|
|
|
|
ensure_space() {
|
|
local path=$1 additional=${2:-0} avail name dir candidates code protected=''
|
|
[[ $additional =~ ^(0|[1-9][0-9]{0,15})$ ]] || die 'Invalid or excessive space request'
|
|
local required=$((MIN_FREE_GB * 1024 * 1024 * 1024 + additional))
|
|
avail=$(available_space "$path")
|
|
(( avail < required )) || return 0
|
|
echo "-- Space recovery: need $required bytes, available $avail on $path"
|
|
echo "-- Space policy: keep $SPACE_KEEP_LOCAL uploaded local set(s) under pressure; normal retention keeps $RETENTION_KEEP_LAST"
|
|
if [[ $ENABLE_UPLOAD == true && $(stat -c %d "$path") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
|
|
check_remote
|
|
resume_pending_uploads
|
|
candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
|
|
protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL")
|
|
if [[ -n $protected ]]; then
|
|
while IFS= read -r name; do
|
|
echo "-- Space recovery: validating protected local copy: $name"
|
|
if helper verify "$ARCHIVE_DIR/$name" >/dev/null; then continue; else code=$?; fi
|
|
(( code < 128 )) || return "$code"
|
|
echo "WARNING: Protected local copy is damaged; keeping all candidates: $name"
|
|
candidates=''
|
|
protected=''
|
|
break
|
|
done <<< "$protected"
|
|
fi
|
|
while IFS= read -r name; do
|
|
[[ -n $name ]] || continue
|
|
avail=$(available_space "$path")
|
|
(( avail < required )) || return 0
|
|
dir="$ARCHIVE_DIR/$name"
|
|
# A manually selected local restore source is never a cleanup candidate.
|
|
[[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
|
|
echo "-- Space recovery: verifying offsite copy before local removal: $name"
|
|
if helper verify "$dir" >/dev/null &&
|
|
remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
|
|
remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
|
|
python3 "$LIB_DIR/space.py" remove "$dir"
|
|
echo "-- Space recovery: removed local copy $name; offsite copy with matching file sizes retained"
|
|
else
|
|
code=$?
|
|
(( code < 128 )) || return "$code"
|
|
echo "WARNING: Keeping $name: local/offsite verification failed"
|
|
fi
|
|
avail=$(available_space "$path")
|
|
if (( avail >= required )); then
|
|
echo "-- Space recovery complete: available $avail bytes, required $required"
|
|
return 0
|
|
fi
|
|
done <<< "$candidates"
|
|
release_local_floor "$path" "$required" "$protected"
|
|
fi
|
|
avail=$(available_space "$path")
|
|
(( avail < required )) || return 0
|
|
python3 "$LIB_DIR/space.py" inventory "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL" ||
|
|
echo 'WARNING: Local archive inventory incomplete; no further cleanup attempted'
|
|
die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: remote verification/quota failed, protected data or workspace capacity prevents further safe cleanup."
|
|
}
|
|
|
|
prepare_backup_space() {
|
|
if [[ -f $WORKDIR/restore-test/.owner ]]; then
|
|
local cleanup_script="$LIB_DIR/../usr-local-sbin/app-restore-test.sh"
|
|
[[ -f $cleanup_script ]] || cleanup_script=/usr/local/sbin/app-restore-test.sh
|
|
APP_BACKUP_INHERIT_LOCK=1 bash "$cleanup_script" --config "$CONFIG_FILE" --cleanup
|
|
fi
|
|
clean_backup_scratch
|
|
local i stats bytes count total=0 db_size
|
|
# Dry-run obeys the exact copy exclusions. No dump, copy or service stop.
|
|
for i in "${!LABELS[@]}"; do
|
|
check_source_path "${SOURCES[i]}"
|
|
exclusions "${LABELS[i]}"
|
|
stats=$(rsync -aHAX --dry-run --stats --numeric-ids "${EXCLUDES[@]}" -- "${SOURCES[i]}/" "$STAGING_ROOT/.space-estimate/" 8>&- 9>&-)
|
|
bytes=$(awk '/^Total file size:/ {gsub(/,/, "", $4); print $4}' <<< "$stats")
|
|
count=$(awk '/^Number of files:/ {gsub(/,/, "", $4); print $4}' <<< "$stats")
|
|
[[ $bytes =~ ^[0-9]+$ && $count =~ ^[0-9]+$ ]] || die 'Cannot estimate source allocation'
|
|
total=$((total + bytes + count * 8192))
|
|
done
|
|
for i in "${!DB_NAMES[@]}"; do
|
|
secure_file "${DB_CNFS[i]}"
|
|
db_size=$("$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --batch --skip-column-names --execute='SELECT COALESCE(SUM(data_length+index_length),0) FROM information_schema.tables WHERE table_schema=DATABASE()')
|
|
[[ $db_size =~ ^[0-9]+$ ]] || die 'Cannot estimate database size'
|
|
total=$((total + db_size * 3 + 67108864))
|
|
done
|
|
# Account for SQL expansion, tar metadata and growth between inventory/copy.
|
|
total=$((total * 12 / 10 + 67108864))
|
|
echo "-- Estimated snapshot budget: $total bytes (+ archive overlap and free-space reserve)"
|
|
if [[ $(stat -c %d "$STAGING_ROOT") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
|
|
ensure_space "$STAGING_ROOT" "$((total * 2))"
|
|
else
|
|
ensure_space "$STAGING_ROOT" "$total"
|
|
ensure_space "$ARCHIVE_DIR" "$total"
|
|
fi
|
|
}
|
|
|
|
# Bound listing/quota/control-file operations just like remote verification.
|
|
space_rc() {
|
|
local attempt code=1
|
|
for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
|
|
if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
|
|
"$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@" \
|
|
--timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES"; then
|
|
return 0
|
|
else code=$?; fi
|
|
(( code < 128 )) || return "$code"
|
|
if (( attempt < RCLONE_CHECK_ATTEMPTS )); then sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?; fi
|
|
done
|
|
return "$code"
|
|
}
|
|
|
|
remote_free_space() {
|
|
local -a statuses
|
|
if space_rc about "$RCLONE_REMOTE_BASE" --json | python3 "$LIB_DIR/space.py" quota; then return 0; else
|
|
statuses=("${PIPESTATUS[@]}")
|
|
if (( statuses[0] != 0 )); then return "${statuses[0]}"; fi
|
|
return "${statuses[1]}"
|
|
fi
|
|
}
|
|
|
|
# Read only small control files and the decrypted size inventory. Identity must
|
|
# match the directory, and every archive must exist with its manifest size.
|
|
space_remote_valid() {
|
|
local name=$1 folder=$2 remote="${RCLONE_REMOTE_BASE%/}/$1"
|
|
space_rc cat "$remote/COMPLETE" --head 256 > "$folder/COMPLETE" &&
|
|
space_rc cat "$remote/manifest.json" --head 1048577 > "$folder/manifest.json" &&
|
|
space_rc lsjson "$remote" --max-depth 1 > "$folder/listing.json" &&
|
|
python3 "$LIB_DIR/space.py" remote-valid "$folder" "$name"
|
|
}
|
|
|
|
ensure_remote_space() (
|
|
local additional=$1 exclude=${2:-} free scratch name anchor code kept=0
|
|
additional=$((additional + additional / 20 + 1048576))
|
|
[[ $ENABLE_UPLOAD == true && $SPACE_REMOTE_CLEANUP == true ]] || return 0
|
|
if free=$(remote_free_space); then :; else
|
|
code=$?
|
|
(( code < 128 )) || return "$code"
|
|
echo 'WARNING: Remote quota unavailable; no automatic remote pressure cleanup (normal upload will still be attempted)'
|
|
return 0
|
|
fi
|
|
(( free < additional )) || return 0
|
|
echo "-- Remote space recovery: need $additional bytes, available $free; keep $SPACE_KEEP_REMOTE complete offsite set(s)"
|
|
scratch=$(mktemp -d "$STAGING_ROOT/.remote-space.XXXXXX") || return 1
|
|
trap 'rm -rf -- "$scratch"' EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
if space_rc lsf "$RCLONE_REMOTE_BASE" --dirs-only --max-depth 1 > "$scratch/dirs"; then :; else
|
|
code=$?
|
|
# No deletion on an incomplete listing. Keep cancellation semantics.
|
|
rm -rf -- "$scratch"
|
|
return "$code"
|
|
fi
|
|
: > "$scratch/valid" || return 1
|
|
# Newest first: quota cleanup always protects the newest verified offsite sets.
|
|
sort -r "$scratch/dirs" > "$scratch/sorted" || return 1
|
|
while IFS= read -r name; do
|
|
name=${name%/}
|
|
[[ $name =~ ^${ARCHIVE_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}_[0-9a-f]{8}$ ]] || continue
|
|
[[ $name != "$exclude" && $name != "${REMOTE_RUN:-}" ]] || continue
|
|
if space_remote_valid "$name" "$scratch"; then
|
|
printf '%s\n' "$name" >> "$scratch/valid" || return 1
|
|
else
|
|
code=$?
|
|
if (( code >= 128 )); then rm -rf -- "$scratch"; return "$code"; fi
|
|
echo "WARNING: Remote pressure cleanup preserves unverified/incomplete set: $name"
|
|
fi
|
|
done < "$scratch/sorted"
|
|
: > "$scratch/keep" || return 1
|
|
: > "$scratch/candidates" || return 1
|
|
while IFS= read -r name; do
|
|
if (( kept < SPACE_KEEP_REMOTE )); then
|
|
echo "-- Remote pressure cleanup protects: $name"
|
|
printf '%s\n' "$name" >> "$scratch/keep" || return 1
|
|
kept=$((kept + 1))
|
|
else printf '%s\n' "$name" >> "$scratch/candidates" || return 1; fi
|
|
done < "$scratch/valid"
|
|
sort "$scratch/candidates" > "$scratch/delete" || return 1
|
|
while IFS= read -r name; do
|
|
while IFS= read -r anchor; do
|
|
if space_remote_valid "$anchor" "$scratch"; then :; else
|
|
code=$?; rm -rf -- "$scratch"; return "$code"
|
|
fi
|
|
done < "$scratch/keep"
|
|
if space_remote_valid "$name" "$scratch"; then :; else
|
|
code=$?; rm -rf -- "$scratch"; return "$code"
|
|
fi
|
|
echo "-- Remote pressure cleanup: removing old complete set $name"
|
|
if space_rc purge "${RCLONE_REMOTE_BASE%/}/$name"; then :; else
|
|
code=$?; rm -rf -- "$scratch"; return "$code"
|
|
fi
|
|
# Provider quota/recycle-bin accounting can lag: never assume purge freed it.
|
|
if free=$(remote_free_space); then :; else
|
|
code=$?; rm -rf -- "$scratch"; return "$code"
|
|
fi
|
|
if (( free >= additional )); then rm -rf -- "$scratch"; return 0; fi
|
|
done < "$scratch/delete"
|
|
rm -rf -- "$scratch"
|
|
echo "ERROR: Insufficient remote quota after safe cleanup: need $additional bytes, available $free; newest offsite backups retained"
|
|
return 1
|
|
)
|
|
|
|
resume_pending_uploads() {
|
|
local name dir code size
|
|
local pending
|
|
pending=$(python3 "$LIB_DIR/space.py" pending "$ARCHIVE_DIR" "$ARCHIVE_PREFIX") || return 1
|
|
while IFS= read -r name; do
|
|
[[ -n $name ]] || continue
|
|
dir="$ARCHIVE_DIR/$name"
|
|
[[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
|
|
echo "-- Space recovery: retrying pending publication: $name"
|
|
if helper verify "$dir" >/dev/null; then :; else
|
|
code=$?; (( code < 128 )) || return "$code"
|
|
echo "WARNING: Keeping invalid pending local set: $name"; continue
|
|
fi
|
|
if remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
|
|
remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
|
|
date -u -Is > "$dir/UPLOADED" || return 1
|
|
sync -f "$dir" || return 1
|
|
echo "-- Pending publication already complete offsite: $name"
|
|
continue
|
|
else
|
|
code=$?; (( code < 128 )) || return "$code"
|
|
fi
|
|
size=$(du -sb -- "$dir" | awk '{print $1}') || return 1
|
|
if ensure_remote_space "$size" "$name" &&
|
|
rc copy "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /COMPLETE --exclude /UPLOADED --immutable &&
|
|
remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /COMPLETE --exclude /UPLOADED &&
|
|
rc copyto "$dir/COMPLETE" "${RCLONE_REMOTE_BASE%/}/$name/COMPLETE" --immutable &&
|
|
remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
|
|
date -u -Is > "$dir/UPLOADED" || return 1
|
|
sync -f "$dir" || return 1
|
|
echo "-- Pending publication confirmed: $name"
|
|
else
|
|
code=$?; (( code < 128 )) || return "$code"
|
|
echo "WARNING: Pending upload remains protected: $name"
|
|
fi
|
|
done <<< "$pending"
|
|
}
|
|
|
|
release_local_floor() {
|
|
local path=$1 required=$2 protected=$3 name dir code bytes available reclaimable=0
|
|
[[ $SPACE_ALLOW_REMOTE_ONLY == true && -n $protected ]] || return 0
|
|
# Do not sacrifice the local floor if its allocated blocks cannot solve the
|
|
# deficit. This also avoids deleting the last local set on undersized disks.
|
|
available=$(available_space "$path") || return 1
|
|
while IFS= read -r name; do
|
|
dir="$ARCHIVE_DIR/$name"
|
|
[[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
|
|
bytes=$(du -sx -B1 -- "$dir" | awk '{print $1}') || return 1
|
|
reclaimable=$((reclaimable + bytes))
|
|
done <<< "$protected"
|
|
(( available + reclaimable >= required )) || return 0
|
|
echo '-- Emergency space recovery: allowing offsite-only history until the new backup completes'
|
|
# protected is newest first. Release oldest first, stopping at the target.
|
|
while IFS= read -r name; do
|
|
dir="$ARCHIVE_DIR/$name"
|
|
[[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
|
|
# ensure_space already validated all protected local contents under the lock.
|
|
if remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED &&
|
|
remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then
|
|
python3 "$LIB_DIR/space.py" remove "$dir" || return 1
|
|
echo "-- Emergency space recovery: removed local copy $name; verified offsite set retained"
|
|
else
|
|
code=$?; (( code < 128 )) || return "$code"
|
|
echo "WARNING: Emergency cleanup keeps $name: local/offsite verification failed"
|
|
fi
|
|
available=$(available_space "$path") || return 1
|
|
(( available < required )) || return 0
|
|
done < <(sort <<< "$protected")
|
|
}
|