Add monthly offsite restore tests and automatic cleanup

usr-local-sbin/app-restore-test.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Read trusted config and hold the same lock as backup/restore/update.
+set -Eeuo pipefail
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+LIB_DIR="$SCRIPT_DIR/../lib/app-backup"
+[[ -f $LIB_DIR/common.sh ]] || LIB_DIR="$SCRIPT_DIR/../lib"
+# shellcheck source=lib/common.sh
+source "$LIB_DIR/common.sh"
+CONFIG_FILE=/etc/app-backup/app-backup.conf
+MODE=run
+REMOTE_RUN=''
+while (( $# )); do
+  case $1 in
+    --config) CONFIG_FILE=${2:?Missing config}; shift 2 ;;
+    --check) MODE=check; shift ;;
+    --cleanup) MODE=cleanup; shift ;;
+    --remote-run) REMOTE_RUN=${2:?Missing run}; shift 2 ;;
+    --help) echo 'Usage: app-restore-test.sh [--config FILE] [--check | --cleanup] [--remote-run NAME]'; exit 0 ;;
+    *) die "Unknown argument: $1" ;;
+  esac
+done
+load_config
+# Cleanup must still work if the feature was disabled after an interrupted test.
+[[ $MODE != run || $ENABLE_RESTORE_TEST == true ]] || die 'Set ENABLE_RESTORE_TEST=true after preparing the test image and disk budget'
+guard_workspaces
+# shellcheck source=lib/locking.sh
+source "$LIB_DIR/locking.sh"
+acquire_backup_lock 75 || exit $?
+args=("$MODE" --workdir "$WORKDIR" --prefix "$ARCHIVE_PREFIX" --remote "$RCLONE_REMOTE_BASE"
+  --run "$REMOTE_RUN" --max-age-hours "$RESTORE_TEST_MAX_AGE_HOURS"
+  --max-work-gb "$RESTORE_TEST_MAX_WORK_GB" --min-free-gb "$RESTORE_TEST_MIN_FREE_GB"
+  --timeout "$RESTORE_TEST_TIMEOUT" --image "$RESTORE_TEST_DB_IMAGE"
+  --hostname "$MAIL_HOSTNAME" --to "$MAIL_TO" --sender "$MAIL_FROM"
+  --sendmail "$MAIL_SENDMAIL_BIN" --mail-timeout "$MAIL_SEND_TIMEOUT" --mail "$ENABLE_MAIL_REPORT")
+if [[ $MODE != cleanup ]]; then
+  build_plan
+  for label in meta "${LABELS[@]}" "${DB_LABELS[@]}"; do args+=(--expected "$label"); done
+  # For run failures, let the Python supervisor report preflight errors as well.
+  args+=(--remote-check "$LIB_DIR/restore-test-remote.sh" --config "$CONFIG_FILE")
+fi
+exec python3 "$LIB_DIR/restore_test.py" "${args[@]}"