Prepare MariaDB restore tests and recover backup disk space safely

usr-local-sbin/app-backup.sh

@@ -9,12 +9,14 @@ source "$LIB_DIR/common.sh"
 CONFIG_FILE=/etc/app-backup/app-backup.conf
 CHECK=false
 SKIP_RETENTION=false
+PREPARE_SPACE=false
 while (( $# )); do
   case $1 in
     --config) CONFIG_FILE=${2:?Missing config path}; shift 2 ;;
     --check) CHECK=true; shift ;;
+    --prepare-space) PREPARE_SPACE=true; CHECK=true; shift ;;
     --no-retention) SKIP_RETENTION=true; shift ;;
-    --help) echo 'Usage: app-backup.sh [--config FILE] [--check] [--no-retention]'; exit 0 ;;
+    --help) echo 'Usage: app-backup.sh [--config FILE] [--check | --prepare-space] [--no-retention]'; exit 0 ;;
     *) die "Unknown argument: $1" ;;
   esac
 done
@@ -22,6 +24,9 @@ OPERATION=backup
 load_config
 init_runtime
 build_plan
+# shellcheck source=lib/space.sh
+source "$LIB_DIR/space.sh"
+if [[ $CHECK == false || $PREPARE_SPACE == true ]]; then prepare_backup_space; fi
 preflight
 if [[ $CHECK == true ]]; then
   echo 'Preflight passed. Directory metadata probed; no application backup, service stop or upload performed.'
@@ -75,6 +80,9 @@ assert_services_quiescent
 echo '-- Copy window complete; writer guards verified'
 BACKUP_PHASE='Dienste wiederherstellen'
 resume_services || die 'Failed to resume original service state'
+if [[ $ENABLE_MAIL_REPORT == true ]]; then
+  python3 "$LIB_DIR/report.py" snapshot "$LOG_DIR/report_$RUN_ID.sizes.json" "$STAGING_DIR" "$PARTIAL_DIR" || echo 'WARNING: Report sizes unavailable'
+fi
 
 while IFS=$'\t' read -r label _; do
   if [[ $label == meta ]]; then root=meta
@@ -93,6 +101,9 @@ while IFS=$'\t' read -r label _; do
       gzip -6 -c > "$out.partial"
   fi
   mv -- "$out.partial" "$out"
+  helper tar "$out" "$root"
+  # Retain small metadata/plan for publication and recognized crash recovery.
+  if [[ $label != meta ]]; then rm -rf -- "${STAGING_DIR:?}/$root"; fi
 done < "$PLAN"
 BACKUP_PHASE='Lokale Archivprüfung'
 helper create "$PARTIAL_DIR" "$RUN_ID" "$PLAN"

usr-local-sbin/app-restore-test.sh

@@ -14,6 +14,7 @@ while (( $# )); do
     --config) CONFIG_FILE=${2:?Missing config}; shift 2 ;;
     --check) MODE=check; shift ;;
     --cleanup) MODE=cleanup; shift ;;
+    --prepare-db) MODE=prepare; shift ;;
     --remote-run) REMOTE_RUN=${2:?Missing run}; shift 2 ;;
     --help) echo 'Usage: app-restore-test.sh [--config FILE] [--check | --cleanup] [--remote-run NAME]'; exit 0 ;;
     *) die "Unknown argument: $1" ;;

usr-local-sbin/app-restore.sh

@@ -38,18 +38,71 @@ done
 OPERATION=restore
 load_config
 init_runtime
-check_space "$RESTORE_ROOT"
+# shellcheck source=lib/space.sh
+source "$LIB_DIR/space.sh"
+clean_backup_scratch
+ensure_space "$RESTORE_ROOT"
 STAGING_DIR=$(mktemp -d "$RESTORE_ROOT/${RUN_ID}.XXXXXX")
+if [[ $MODE == verify ]]; then printf '%s\n' "$RUN_ID" > "$STAGING_DIR/.verify-only"; fi
 INPUT="$STAGING_DIR/input"
 mkdir -m 700 "$INPUT"
 if [[ -n $REMOTE_RUN ]]; then
   [[ $REMOTE_RUN =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]+$ && $REMOTE_RUN == "$ARCHIVE_PREFIX"_* ]] || die 'Unsafe remote run name'
   check_remote
+  if [[ $LEGACY == false ]]; then
+    remote="${RCLONE_REMOTE_BASE%/}/$REMOTE_RUN"
+    rc cat "$remote/manifest.json" --count 1048577 > "$INPUT/manifest.json"
+    rc cat "$remote/COMPLETE" --count 256 > "$INPUT/COMPLETE"
+    helper metadata "$INPUT" >/dev/null
+    python3 - "$INPUT" "$REMOTE_RUN" <<'PY' > "$STAGING_DIR/verify.tsv"
+import json,sys
+from pathlib import Path
+data=json.loads((Path(sys.argv[1])/'manifest.json').read_text())
+assert data['run_id'] == sys.argv[2], 'Remote run ID mismatch'
+for item in data['archives']:
+    print('\t'.join(str(item[k]) for k in ('file','root','size','sha256')))
+PY
+    rc lsf "$remote" --files-only --max-depth 1 > "$STAGING_DIR/remote-files.txt"
+    python3 - "$INPUT/manifest.json" "$STAGING_DIR/remote-files.txt" <<'PY'
+import json,sys
+from pathlib import Path
+expected={x['file'] for x in json.loads(Path(sys.argv[1]).read_text())['archives']} | {'manifest.json','COMPLETE'}
+actual=set(Path(sys.argv[2]).read_text().splitlines())
+assert actual in (expected, expected | {'UPLOADED'}), 'Remote backup has missing or unexpected files'
+PY
+  fi
+  if [[ $MODE == verify ]]; then
+    # Verify offsite archives sequentially, retaining at most one compressed file.
+    while IFS=$'\t' read -r file root size hash; do
+      ensure_space "$RESTORE_ROOT" "$size"
+      rc cat "$remote/$file" --count "$((size + 1))" > "$INPUT/$file"
+      [[ $(stat -c %s "$INPUT/$file") == "$size" && $(sha256sum "$INPUT/$file" | cut -d ' ' -f 1) == "$hash" ]] || die "Archive checksum/size mismatch: $file"
+      helper tar "$INPUT/$file" "$root"
+      rm -f -- "$INPUT/$file"
+    done < "$STAGING_DIR/verify.tsv"
+    echo 'Verification passed; remote archives checked sequentially; no application changes.'
+    exit 0
+  fi
+  if [[ $LEGACY == false ]]; then
+    size=$(python3 - "$INPUT/manifest.json" <<'PY'
+import json,sys
+print(sum(x['size'] for x in json.load(open(sys.argv[1]))['archives']))
+PY
+)
+    ensure_space "$RESTORE_ROOT" "$size"
+  fi
   echo "-- Downloading remote backup for restore verification: $REMOTE_RUN"
   rc copy "${RCLONE_REMOTE_BASE%/}/$REMOTE_RUN" "$INPUT"
   echo '-- Remote backup downloaded; validating manifest, hashes and archives'
 else
   [[ -d $LOCAL_RUN && ! -L $LOCAL_RUN ]] || die 'Local run must be an existing directory'
+  if [[ $MODE == verify ]]; then
+    helper verify "$LOCAL_RUN"
+    echo 'Verification passed; local archives checked without copying; no application changes.'
+    exit 0
+  fi
+  size=$(du -s -B1 -- "$LOCAL_RUN" | awk '{print $1}')
+  ensure_space "$RESTORE_ROOT" "$size"
   # Resolve overlap separately: archive sources are allowed under WORKDIR, but
   # recursive copies into the freshly created restore workspace are forbidden.
   input_real=$(realpath -e "$LOCAL_RUN")
@@ -85,6 +138,8 @@ helper verify "$INPUT"
 if [[ $MODE == verify ]]; then echo 'Verification passed; no application changes.'; exit 0; fi
 helper list "$INPUT" > "$STAGING_DIR/archive-list.tsv"
 if [[ $MODE == extract ]]; then
+  size=$(helper expanded "$INPUT")
+  ensure_space "$RESTORE_ROOT" "$size"
   mkdir "$STAGING_DIR/extracted"
   while IFS=$'\t' read -r label file _; do
     mkdir "$STAGING_DIR/extracted/$label"
@@ -137,6 +192,9 @@ if [[ $MODE == plan ]]; then
   exit 0
 fi
 mkdir "$STAGING_DIR/extracted"
+# Budget all extracted components before productive writer shutdown/mutation.
+size=$(helper expanded "$INPUT")
+ensure_space "$RESTORE_ROOT" "$size"
 # Each component gets an independent extraction root, so links in one component
 # cannot redirect extraction of a later component into another tree.
 while IFS=$'\t' read -r label file _; do