#!/usr/bin/env bash # Called only under the shared lock; --check never invokes this maintenance. clean_backup_scratch() { python3 "$LIB_DIR/space.py" clean "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$RESTORE_ROOT" } available_space() { local value value=$(df -PB1 -- "$1" | awk 'END{print $4}') || return 1 [[ $value =~ ^(0|[1-9][0-9]{0,17})$ ]] || die "Cannot determine available space on $1" printf '%s\n' "$value" } ensure_space() { local path=$1 additional=${2:-0} avail name dir candidates code protected [[ $additional =~ ^(0|[1-9][0-9]{0,15})$ ]] || die 'Invalid or excessive space request' local required=$((MIN_FREE_GB * 1024 * 1024 * 1024 + additional)) avail=$(available_space "$path") (( avail < required )) || return 0 echo "-- Space recovery: need $required bytes, available $avail on $path" echo "-- Space policy: keep $SPACE_KEEP_LOCAL uploaded local set(s) under pressure; normal retention keeps $RETENTION_KEEP_LAST" if [[ $ENABLE_UPLOAD == true && $(stat -c %d "$path") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then check_remote candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL") if [[ -n $candidates ]]; then protected=$(python3 "$LIB_DIR/space.py" protected "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL") [[ -n $protected ]] || die 'No protected local backup; refusing space recovery' while IFS= read -r name; do echo "-- Space recovery: validating protected local copy: $name" if helper verify "$ARCHIVE_DIR/$name" >/dev/null; then continue; else code=$?; fi (( code < 128 )) || return "$code" echo "WARNING: Protected local copy is damaged; keeping all candidates: $name" candidates='' break done <<< "$protected" fi while IFS= read -r name; do [[ -n $name ]] || continue avail=$(available_space "$path") (( avail < required )) || return 0 dir="$ARCHIVE_DIR/$name" # A manually selected local restore source is never a cleanup candidate. [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue echo "-- Space recovery: verifying offsite copy before local removal: $name" if helper verify "$dir" >/dev/null && remote_metadata_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --exclude /UPLOADED && remote_identity_check "$dir" "${RCLONE_REMOTE_BASE%/}/$name"; then python3 "$LIB_DIR/space.py" remove "$dir" echo "-- Space recovery: removed local copy $name; offsite copy with matching file sizes retained" else code=$? (( code < 128 )) || return "$code" echo "WARNING: Keeping $name: local/offsite verification failed" fi avail=$(available_space "$path") if (( avail >= required )); then echo "-- Space recovery complete: available $avail bytes, required $required" return 0 fi done <<< "$candidates" fi avail=$(available_space "$path") (( avail < required )) || return 0 python3 "$LIB_DIR/space.py" inventory "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$SPACE_KEEP_LOCAL" || echo 'WARNING: Local archive inventory incomplete; no further cleanup attempted' die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail, deficit $((required - avail)) bytes; protected backups/recovery data retained. See SPEICHER.md: resolve pending uploads or move backup workspace to a larger filesystem." } prepare_backup_space() { if [[ -f $WORKDIR/restore-test/.owner ]]; then local cleanup_script="$LIB_DIR/../usr-local-sbin/app-restore-test.sh" [[ -f $cleanup_script ]] || cleanup_script=/usr/local/sbin/app-restore-test.sh APP_BACKUP_INHERIT_LOCK=1 bash "$cleanup_script" --config "$CONFIG_FILE" --cleanup fi clean_backup_scratch local i stats bytes count total=0 db_size # Dry-run obeys the exact copy exclusions. No dump, copy or service stop. for i in "${!LABELS[@]}"; do check_source_path "${SOURCES[i]}" exclusions "${LABELS[i]}" stats=$(rsync -aHAX --dry-run --stats --numeric-ids "${EXCLUDES[@]}" -- "${SOURCES[i]}/" "$STAGING_ROOT/.space-estimate/" 8>&- 9>&-) bytes=$(awk '/^Total file size:/ {gsub(/,/, "", $4); print $4}' <<< "$stats") count=$(awk '/^Number of files:/ {gsub(/,/, "", $4); print $4}' <<< "$stats") [[ $bytes =~ ^[0-9]+$ && $count =~ ^[0-9]+$ ]] || die 'Cannot estimate source allocation' total=$((total + bytes + count * 8192)) done for i in "${!DB_NAMES[@]}"; do secure_file "${DB_CNFS[i]}" db_size=$("$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --batch --skip-column-names --execute='SELECT COALESCE(SUM(data_length+index_length),0) FROM information_schema.tables WHERE table_schema=DATABASE()') [[ $db_size =~ ^[0-9]+$ ]] || die 'Cannot estimate database size' total=$((total + db_size * 3 + 67108864)) done # Account for SQL expansion, tar metadata and growth between inventory/copy. total=$((total * 12 / 10 + 67108864)) echo "-- Estimated snapshot budget: $total bytes (+ archive overlap and free-space reserve)" if [[ $(stat -c %d "$STAGING_ROOT") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then ensure_space "$STAGING_ROOT" "$((total * 2))" else ensure_space "$STAGING_ROOT" "$total" ensure_space "$ARCHIVE_DIR" "$total" fi }