Prepare MariaDB restore tests and recover backup disk space safely

lib/backup_set.py

@@ -99,9 +99,11 @@ def check_tar(path, roots):
     proc = subprocess.Popen(command, stdout=subprocess.PIPE)
     entries = {}
     hardlinks = []
+    expanded = 0
     try:
         with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
             for item in archive:
+                expanded += ((item.size + 4095) // 4096) * 4096 + 4096
                 name = member_name(item.name)
                 if not any(name == root or name.startswith(root + "/") for root in roots):
                     raise ValueError(f"Member outside component root: {name}")
@@ -128,6 +130,7 @@ def check_tar(path, roots):
             pass
         if proc.wait() != 0:
             raise ValueError(f"Decompression failed: {path}")
+        return expanded
     finally:
         proc.stdout.close()
         if proc.poll() is None:
@@ -291,9 +294,11 @@ def main():
         crypt(*args)
     elif op == "create":
         create(Path(args[0]), *args[1:])
-    elif op in ("verify", "metadata", "list"):
+    elif op in ("verify", "metadata", "list", "expanded"):
         data = load_set(Path(args[0]), metadata_only=op == "metadata")
-        if op == "verify":
+        if op == "expanded":
+            print(sum(check_tar(Path(args[0]) / item['file'], [item['root']]) for item in data['archives']))
+        elif op == "verify":
             for item in data["archives"]:
                 check_tar(Path(args[0]) / item["file"], [item["root"]])
             print(f"Verified: {data['run_id']} ({len(data['archives'])} archives)")

lib/common.sh

@@ -40,7 +40,7 @@ load_config() {
   : "${MAIL_HOSTNAME:=}"
   : "${MAIL_SENDMAIL_BIN:=/usr/sbin/sendmail}" "${MAIL_SEND_TIMEOUT:=30}"
   : "${ENABLE_RESTORE_TEST:=false}" "${RESTORE_TEST_DB_IMAGE:=}"
-  : "${RESTORE_TEST_MAX_WORK_GB:=4}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
+  : "${RESTORE_TEST_MAX_WORK_GB:=0}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
   : "${RESTORE_TEST_MAX_AGE_HOURS:=72}" "${RESTORE_TEST_TIMEOUT:=39600}"
   : "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
   : "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
@@ -71,6 +71,7 @@ load_config() {
     [[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
   done
   (( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
+  (( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
   (( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
   [[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
   [[ $COMPRESSOR == zstd || $COMPRESSOR == gzip ]] || die "COMPRESSOR must be zstd or gzip"
@@ -487,6 +488,7 @@ resume_services() {
 }
 finish() {
   local code=$?
+  local recovery_ok=true
   trap - EXIT INT TERM
   set +e
   if [[ $OPERATION == restore && $MUTATION_STARTED == true && $code != 0 ]]; then
@@ -495,7 +497,7 @@ finish() {
     echo "ERROR: Partial restore. Service recovery requires manual review: ${STOPPED_SERVICES[*]}"
     echo "ERROR: Keep Nextcloud in maintenance; review $LOG_FILE before restarting."
   else
-    resume_services || code=1
+    if ! resume_services; then code=1; recovery_ok=false; fi
   fi
   local report_sizes="$LOG_DIR/report_$RUN_ID.sizes.json"
   if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
@@ -503,7 +505,18 @@ finish() {
     python3 "$LIB_DIR/report.py" snapshot "$report_sizes" "${STAGING_DIR:-}" "${RUN_DIR:-${PARTIAL_DIR:-}}" ||
       echo 'WARNING: Report sizes unavailable'
   fi
-  if [[ $code == 0 && -n ${STAGING_DIR:-} ]]; then rm -rf -- "$STAGING_DIR" || code=1; fi
+  if [[ $OPERATION == backup && $recovery_ok == true && -n ${STAGING_DIR:-} && ! -e $WORKDIR/recovery-$RUN_ID.json ]]; then
+    # A failed backup scratch copy is disposable once writer recovery succeeded.
+    # Never delete a locally complete set awaiting upload or a completion marker.
+    if [[ -n ${PARTIAL_DIR:-} && -d $PARTIAL_DIR && ! -e $PARTIAL_DIR/COMPLETE ]]; then
+      python3 "$LIB_DIR/space.py" remove "$PARTIAL_DIR" || code=1
+    fi
+    if [[ ! -d ${PARTIAL_DIR:-/nonexistent} ]]; then
+      python3 "$LIB_DIR/space.py" remove "$STAGING_DIR" && STAGING_DIR='' || code=1
+    fi
+  elif [[ ( $code == 0 || ( $OPERATION == restore && ${MODE:-} == verify ) ) && -n ${STAGING_DIR:-} ]]; then
+    rm -rf -- "$STAGING_DIR" && STAGING_DIR='' || code=1
+  fi
   if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi
   # Close the logger's writers and wait for EOF so the mail tail includes the
   # final recovery/status messages. Reporting itself uses the original stdout.

lib/prepare-restore-test.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Root-only installer preparation. No production SQL writes or credentials in Docker.
+set -Eeuo pipefail
+LIB_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=lib/common.sh
+source "$LIB_DIR/common.sh"
+CONFIG_FILE=${1:?Configuration required}
+load_config
+[[ $ENABLE_RESTORE_TEST == true ]] || exit 0
+guard_workspaces
+# shellcheck source=lib/locking.sh
+source "$LIB_DIR/locking.sh"
+acquire_backup_lock 1
+export APP_BACKUP_INHERIT_LOCK=1
+build_plan
+(( ${#DB_NAMES[@]} )) || exit 0
+need docker; need timeout
+docker_cmd=(docker --host=unix:///var/run/docker.sock)
+if ! "${docker_cmd[@]}" info --format '{{.DockerRootDir}}' >/dev/null 2>&1; then
+  systemctl start docker.service
+fi
+docker_root=$("${docker_cmd[@]}" info --format '{{.DockerRootDir}}')
+[[ $docker_root == /* ]] || die 'Cannot determine local Docker storage'
+version=''
+for i in "${!DB_NAMES[@]}"; do
+  secure_file "${DB_CNFS[i]}"
+  server=$("$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --batch --skip-column-names --execute='SELECT VERSION()')
+  [[ $server =~ ^([0-9]+\.[0-9]+)\.[0-9]+.*MariaDB ]] || die 'Automatic restore-test preparation requires MariaDB; no production server changes made'
+  found=${BASH_REMATCH[1]}
+  [[ -z $version || $version == "$found" ]] || die 'Different production MariaDB series require separate restore-test configurations'
+  version=$found
+done
+image=${RESTORE_TEST_DB_IMAGE:-mariadb:$version}
+[[ $image =~ ^[a-zA-Z0-9][a-zA-Z0-9._/:@-]*$ ]] || die 'Invalid restore-test image reference'
+if ! "${docker_cmd[@]}" image inspect "$image" >/dev/null 2>&1; then
+  # Image layers are stored on Docker's filesystem, not necessarily WORKDIR.
+  [[ -d $docker_root ]] || die 'Docker storage is not accessible for the space check'
+  # shellcheck source=lib/space.sh
+  source "$LIB_DIR/space.sh"
+  ensure_space "$docker_root" 2147483648
+  echo "-- Preparing isolated MariaDB image: $image"
+  timeout --kill-after=30s 1800 "${docker_cmd[@]}" pull "$image"
+fi
+# Pin the probe to an immutable local image ID; never pull during monthly jobs.
+image_id=$("${docker_cmd[@]}" image inspect --format '{{.Id}}' "$image")
+"${docker_cmd[@]}" image inspect "$image_id" | python3 -c 'import json,sys; info=json.load(sys.stdin)[0]; assert not set(info.get("Config",{}).get("Volumes") or {}) - {"/var/lib/mysql"}, "Unexpected image volumes"'
+probe=$(timeout --kill-after=10s 60 "${docker_cmd[@]}" run --rm --pull=never --network=none --read-only \
+  --cap-drop=ALL --security-opt=no-new-privileges --memory=128m --pids-limit=64 \
+  --entrypoint=mariadbd "$image_id" --no-defaults --version)
+[[ $probe == *" $version."* && $probe == *MariaDB* ]] || die 'Restore-test image does not match production MariaDB series'
+echo "-- MariaDB $version image verified; testing isolated initialization/import/cleanup"
+# Use a private temporary config; publish the image selection only after success.
+prepared=$(mktemp "${CONFIG_FILE}.prepare-XXXXXX")
+trap 'rm -f -- "$prepared"' EXIT
+cp -- "$CONFIG_FILE" "$prepared"
+printf '\nRESTORE_TEST_DB_IMAGE=%q\n' "$image" >> "$prepared"
+script="$LIB_DIR/../usr-local-sbin/app-restore-test.sh"
+[[ -f $script ]] || script=/usr/local/sbin/app-restore-test.sh
+bash "$script" --config "$prepared" --prepare-db
+if [[ $RESTORE_TEST_DB_IMAGE != "$image" ]]; then
+  chmod 600 "$prepared"
+  mv -fT -- "$prepared" "$CONFIG_FILE"
+fi
+echo '-- MariaDB restore-test preparation passed; temporary database removed'

lib/report.py

@@ -27,7 +27,10 @@ def allocated(path):
 
 def snapshot(output, staging, archives):
     data = {'staged': {}, 'archives': {}}
-    if staging and Path(staging).is_dir():
+    if Path(output).is_file():
+        data.update(json.loads(Path(output).read_text()))
+    captured = 'staging_total' in data
+    if not captured and staging and Path(staging).is_dir():
         root = Path(staging)
         # Measure components independently; the total accounts for shared links.
         paths = sorted((root/'files').glob('*')) + [root/'db', root/'meta']

lib/restore-test-remote.sh

@@ -7,6 +7,18 @@ source "$LIB_DIR/common.sh"
 CONFIG_FILE=${1:?Missing config}
 shift
 load_config
+if [[ ${1:-} == space ]]; then
+  # Local maintenance requested between components, while the parent holds FD 9.
+  [[ /proc/$PPID/fd/9 -ef $LOCKFILE ]] || die 'Space recovery requires the parent backup lock'
+  [[ ${2:-} =~ ^[0-9]+$ && $# == 2 ]] || die 'Invalid space request'
+  guard_workspaces
+  # shellcheck source=lib/space.sh
+  source "$LIB_DIR/space.sh"
+  clean_backup_scratch
+  MIN_FREE_GB=$(python3 -c 'import math,sys; print(math.ceil(float(sys.argv[1])))' "$RESTORE_TEST_MIN_FREE_GB")
+  ensure_space "$WORKDIR" "$2"
+  exit 0
+fi
 check_remote >&2
 # rclone is restricted to read operations here, including metadata reads.
 case ${1:-} in lsf|cat) rc "$@" ;; *) die 'Unsupported restore-test remote operation' ;; esac

lib/restore_test.py

@@ -137,6 +137,19 @@ class Rehearsal:
         if time.monotonic() >= self.deadline:
             raise TestError('Restore-test time limit exceeded')
         free = shutil.disk_usage(self.root).free
+        if (force and self.args.mode == 'run' and self.args.remote_check and
+                free < self.args.min_free_gb * GIB + additional):
+            # Only between subprocesses: do not overwrite the active-child journal.
+            self.log.info('Attempting safe local space recovery before next component step')
+            self.monitor = False
+            try:
+                result = self.remote('space', str(int(additional)), capture=1024*1024)
+                self.log.info('%s', result.decode(errors='replace').strip())
+            except TestError as exc:
+                raise TestError('Free-space reserve unavailable after safe recovery; protected data retained') from exc
+            finally:
+                self.monitor = True
+            free = shutil.disk_usage(self.root).free
         if free < self.args.min_free_gb * GIB + additional:
             raise TestError('Free-space reserve reached; test stopped and scratch will be cleaned')
         if force or time.monotonic() >= self.next_usage:
@@ -145,7 +158,7 @@ class Rehearsal:
                 for name in directories + files:
                     self.used += (Path(path) / name).lstat().st_blocks * 512
             self.next_usage = time.monotonic() + 2
-        if self.used + additional > self.args.max_work_gb * GIB:
+        if self.args.max_work_gb and self.used + additional > self.args.max_work_gb * GIB:
             raise TestError('Restore-test disk budget exceeded')
 
     def command(self, command, *, capture=0, output=None, limit=None, stdin=None, timeout=120):
@@ -459,7 +472,7 @@ def io_bytes(raw):
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('mode', choices=('run', 'check', 'cleanup'))
+    parser.add_argument('mode', choices=('run', 'check', 'cleanup', 'prepare'))
     for key in ('workdir', 'prefix', 'remote', 'run', 'image', 'hostname', 'to', 'sender', 'sendmail', 'mail'):
         parser.add_argument('--' + key, required=True)
     for key in ('remote-check', 'config'):
@@ -470,12 +483,35 @@ def main():
     for key in ('max-work-gb', 'min-free-gb'):
         parser.add_argument('--' + key, type=float, required=True)
     args = parser.parse_args()
-    if not (0 < args.max_work_gb <= 100000 and 0 <= args.min_free_gb <= 100000 and
+    if not (0 <= args.max_work_gb <= 100000 and 0 <= args.min_free_gb <= 100000 and
             1 <= args.max_age_hours <= 8760 and 1 <= args.timeout <= 86400 and 1 <= args.mail_timeout <= 300):
         parser.error('Invalid restore-test limits')
     if os.geteuid() != 0:
         parser.error('Run as root via app-restore-test.sh')
     test = Rehearsal(args)
+    if args.mode == 'prepare':
+        def stop_prepare(signum, _frame):
+            raise TestError('Installer probe interrupted: ' + str(signum))
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            signal.signal(sig, stop_prepare)
+        # Installer smoke test, separate from monthly backup acceptance/status.
+        # Refuse an interrupted rehearsal until its regular cleanup reported it.
+        if any((test.root / name).exists() for name in ('scratch', 'db-created', 'child.json')):
+            raise TestError('Run restore-test --cleanup before DB preparation')
+        try:
+            test.preflight()
+            test.scratch.mkdir(mode=0o700)
+            test.monitor = True
+            sql = test.scratch / 'probe.sql'
+            sql.write_text('CREATE TABLE installer_probe (id INT PRIMARY KEY); INSERT INTO installer_probe VALUES (1);\n')
+            test.disk(additional=512*1024*1024, force=True)
+            if test.database(sql) != 1:
+                raise TestError('Installer database probe returned unexpected table count')
+        finally:
+            test.monitor = False
+            test.cleanup()
+        test.log.info('Installer DB initialization, import, check and cleanup passed')
+        return 0
     if args.mode == 'cleanup' and not test.root.exists():
         return 0
     if args.mode == 'check':

lib/space.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Conservative scratch recovery. Caller MUST hold the shared backup lock.
+
+Only private, recognized backup scratch is disposable. Recovery journals and
+manual restore extractions always win over attempts to reclaim disk space.
+"""
+import argparse
+import os
+from pathlib import Path
+import re
+import shutil
+
+import backup_set
+
+
+def private(path):
+    return (path.is_dir() and path.resolve() == path and
+            path.stat().st_uid == 0 and not path.stat().st_mode & 0o077)
+
+
+def remove_tree(path):
+    if not private(path):
+        raise ValueError('Refusing non-private/aliased scratch: ' + str(path))
+    device = path.stat().st_dev
+    for parent, dirs, _ in os.walk(path, followlinks=False):
+        for name in dirs:
+            child = Path(parent) / name
+            if not child.is_symlink() and (child.stat().st_dev != device or os.path.ismount(child)):
+                raise ValueError('Refusing mounted scratch: ' + str(child))
+    shutil.rmtree(path)  # fd-based, symlink-safe implementation on Linux
+
+
+def clean(work, staging, archives, prefix, restore=None):
+    for path in (work, staging, archives):
+        if not private(path):
+            raise ValueError('Cleanup requires private canonical directories: ' + str(path))
+    if list(work.glob('recovery-*.json')) or list(work.glob('recovery-*.json.tmp')):
+        raise ValueError('Unresolved service/restore recovery journal; preserve all scratch and review services')
+    pattern = re.compile(re.escape(prefix) + r'_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}_[0-9a-f]{8}')
+    for path in sorted(staging.iterdir()):
+        run, dot, suffix = path.name.rpartition('.')
+        if not (dot and re.fullmatch(r'[a-zA-Z0-9]{6}', suffix) and pattern.fullmatch(run) and private(path)):
+            continue
+        identity, version = path/'meta/run-id.txt', path/'meta/format.txt'
+        if any(p.is_symlink() or not p.is_file() or p.stat().st_size > 256 for p in (identity, version)):
+            continue
+        if (path/'meta').is_symlink() or identity.read_text() != run+'\n' or not version.read_text().startswith('format=2\n'):
+            continue
+        partial = archives / ('.partial-' + run)
+        if partial.exists():
+            # A completion marker might be the only surviving complete snapshot.
+            if (partial/'COMPLETE').exists() or (partial/'COMPLETE').is_symlink():
+                continue
+            remove_tree(partial)
+            print('-- Removed abandoned partial archive: ' + partial.name, flush=True)
+        remove_tree(path)
+        print('-- Removed abandoned backup staging: ' + path.name, flush=True)
+    if restore is not None and private(restore):
+        for path in sorted(restore.iterdir()):
+            run, _, suffix = path.name.rpartition('.')
+            marker = path/'.verify-only'
+            if (private(path) and pattern.fullmatch(run) and re.fullmatch(r'[a-zA-Z0-9]{6}', suffix) and
+                    marker.is_file() and not marker.is_symlink() and marker.stat().st_size < 256 and
+                    marker.read_text() == run+'\n'):
+                remove_tree(path)
+                print('-- Removed abandoned verification download: ' + path.name, flush=True)
+
+
+def eligible(archives, prefix, keep):
+    valid = []
+    for path in archives.iterdir():
+        if not private(path) or not path.name.startswith(prefix+'_'):
+            continue
+        marker = path/'UPLOADED'
+        if not marker.is_file() or marker.is_symlink():
+            continue
+        try:
+            manifest = backup_set.load_set(path, metadata_only=True)
+            if manifest['run_id'] != path.name:
+                continue
+        except (ValueError, OSError, KeyError, TypeError):
+            continue
+        valid.append(path.name)
+    # Emergency pruning may ignore age, NEVER the minimum newest-copy count.
+    for name in sorted(valid, reverse=True)[keep:][::-1]:
+        print(name)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('mode', choices=('clean', 'eligible', 'remove'))
+    parser.add_argument('paths', nargs='+')
+    args = parser.parse_args()
+    if args.mode == 'clean':
+        clean(*(Path(p) for p in args.paths[:3]), args.paths[3], Path(args.paths[4]) if len(args.paths) > 4 else None)
+    elif args.mode == 'eligible':
+        eligible(Path(args.paths[0]), args.paths[1], int(args.paths[2]))
+    else:
+        remove_tree(Path(args.paths[0]))

lib/space.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Called only under the shared lock; --check never invokes this maintenance.
+clean_backup_scratch() {
+  python3 "$LIB_DIR/space.py" clean "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$RESTORE_ROOT"
+}
+
+ensure_space() {
+  local path=$1 additional=${2:-0} avail name dir candidates
+  [[ $additional =~ ^(0|[1-9][0-9]{0,15})$ ]] || die 'Invalid or excessive space request'
+  local required=$((MIN_FREE_GB * 1024 * 1024 * 1024 + additional))
+  avail=$(df -PB1 -- "$path" | awk 'END{print $4}')
+  (( avail < required )) || return 0
+  echo "-- Space recovery: need $required bytes, available $avail on $path"
+  if [[ $ENABLE_UPLOAD == true && $(stat -c %d "$path") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
+    check_remote
+    candidates=$(python3 "$LIB_DIR/space.py" eligible "$ARCHIVE_DIR" "$ARCHIVE_PREFIX" "$RETENTION_KEEP_LAST")
+    while IFS= read -r name; do
+      [[ -n $name ]] || continue
+      dir="$ARCHIVE_DIR/$name"
+      # A manually selected local restore source is never a cleanup candidate.
+      [[ -z ${LOCAL_RUN:-} || $(realpath -m "$LOCAL_RUN") != "$dir" ]] || continue
+      echo "-- Space recovery: verifying offsite copy before local removal: $name"
+      if helper verify "$dir" >/dev/null &&
+         rc check "$dir" "${RCLONE_REMOTE_BASE%/}/$name" --download --exclude /UPLOADED; then
+        python3 "$LIB_DIR/space.py" remove "$dir"
+        echo "-- Space recovery: removed local copy $name; verified offsite copy retained"
+      else
+        echo "WARNING: Keeping $name: local/offsite verification failed"
+      fi
+      avail=$(df -PB1 -- "$path" | awk 'END{print $4}')
+      (( avail < required )) || return 0
+    done <<< "$candidates"
+  fi
+  die "Insufficient space after safe cleanup on $path: need $required bytes, available $avail; protected backups/recovery data retained"
+}
+
+prepare_backup_space() {
+  if [[ -f $WORKDIR/restore-test/.owner ]]; then
+    local cleanup_script="$LIB_DIR/../usr-local-sbin/app-restore-test.sh"
+    [[ -f $cleanup_script ]] || cleanup_script=/usr/local/sbin/app-restore-test.sh
+    APP_BACKUP_INHERIT_LOCK=1 bash "$cleanup_script" --config "$CONFIG_FILE" --cleanup
+  fi
+  clean_backup_scratch
+  local i stats bytes count total=0 db_size
+  # Dry-run obeys the exact copy exclusions. No dump, copy or service stop.
+  for i in "${!LABELS[@]}"; do
+    check_source_path "${SOURCES[i]}"
+    exclusions "${LABELS[i]}"
+    stats=$(rsync -aHAX --dry-run --stats --numeric-ids "${EXCLUDES[@]}" -- "${SOURCES[i]}/" "$STAGING_ROOT/.space-estimate/" 8>&- 9>&-)
+    bytes=$(awk '/^Total file size:/ {gsub(/,/, "", $4); print $4}' <<< "$stats")
+    count=$(awk '/^Number of files:/ {gsub(/,/, "", $4); print $4}' <<< "$stats")
+    [[ $bytes =~ ^[0-9]+$ && $count =~ ^[0-9]+$ ]] || die 'Cannot estimate source allocation'
+    total=$((total + bytes + count * 8192))
+  done
+  for i in "${!DB_NAMES[@]}"; do
+    secure_file "${DB_CNFS[i]}"
+    db_size=$("$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --batch --skip-column-names --execute='SELECT COALESCE(SUM(data_length+index_length),0) FROM information_schema.tables WHERE table_schema=DATABASE()')
+    [[ $db_size =~ ^[0-9]+$ ]] || die 'Cannot estimate database size'
+    total=$((total + db_size * 3 + 67108864))
+  done
+  # Account for SQL expansion, tar metadata and growth between inventory/copy.
+  total=$((total * 12 / 10 + 67108864))
+  echo "-- Estimated snapshot budget: $total bytes (+ archive overlap and free-space reserve)"
+  if [[ $(stat -c %d "$STAGING_ROOT") == "$(stat -c %d "$ARCHIVE_DIR")" ]]; then
+    ensure_space "$STAGING_ROOT" "$((total * 2))"
+  else
+    ensure_space "$STAGING_ROOT" "$total"
+    ensure_space "$ARCHIVE_DIR" "$total"
+  fi
+}