Add monthly offsite restore tests and automatic cleanup

install.sh

@@ -17,8 +17,8 @@ while (( $# )); do
 done
 [[ $NO_TIMER != true || $ENABLE_TIMER != true ]] || exit 1
 # Syntax and packaging errors must be caught BEFORE changing the running installation.
-for file in "$ROOT_DIR"/usr-local-sbin/*.sh "$ROOT_DIR/lib/common.sh"; do bash -n "$file"; done
-python3 - "$ROOT_DIR/lib/backup_set.py" "$ROOT_DIR/lib/report.py" <<'PY'
+for file in "$ROOT_DIR"/usr-local-sbin/*.sh "$ROOT_DIR"/lib/*.sh; do bash -n "$file"; done
+python3 - "$ROOT_DIR/lib/backup_set.py" "$ROOT_DIR/lib/report.py" "$ROOT_DIR/lib/restore_test.py" <<'PY'
 import ast,sys
 for path in sys.argv[1:]: ast.parse(open(path).read())
 PY
@@ -32,13 +32,19 @@ if [[ -f /etc/app-backup/app-backup.conf ]]; then
 fi
 # shellcheck source=lib/locking.sh
 source "$ROOT_DIR/lib/locking.sh"
-acquire_backup_lock
+acquire_backup_lock 1
 if systemctl is-active --quiet app-backup.service; then
   echo 'app-backup.service is active. Wait for completion before updating.' >&2
   exit 1
 fi
+if systemctl is-active --quiet app-restore-test.service; then
+  echo 'Restore test is active. Wait for completion before updating.' >&2
+  exit 1
+fi
 TIMER_ACTIVE=false
 if systemctl is-active --quiet app-backup.timer; then TIMER_ACTIVE=true; fi
+RESTORE_TIMER_ACTIVE=false
+if systemctl is-active --quiet app-restore-test.timer; then RESTORE_TIMER_ACTIVE=true; fi
 if [[ -n $SNAPSHOT_DIR ]]; then
   [[ $SNAPSHOT_DIR == /* && ! -e $SNAPSHOT_DIR ]] || { echo 'Snapshot directory must be new and absolute'; exit 1; }
   mkdir -m 700 -- "$SNAPSHOT_DIR"
@@ -49,6 +55,14 @@ declare -a SOURCES TARGETS MODES
 add() { SOURCES+=("$ROOT_DIR/$1"); TARGETS+=("$2"); MODES+=("$3"); }
 add usr-local-sbin/app-backup.sh /usr/local/sbin/app-backup.sh 750
 add usr-local-sbin/app-restore.sh /usr/local/sbin/app-restore.sh 750
+add usr-local-sbin/app-restore-test.sh /usr/local/sbin/app-restore-test.sh 750
+add lib/restore_test.py /usr/local/lib/app-backup/restore_test.py 640
+add lib/restore-test-remote.sh /usr/local/lib/app-backup/restore-test-remote.sh 640
+add systemd/app-restore-test.service /etc/systemd/system/app-restore-test.service 644
+add systemd/app-restore-test-cleanup.service /etc/systemd/system/app-restore-test-cleanup.service 644
+if [[ -e /etc/systemd/system/app-restore-test.timer ]]; then
+  add systemd/app-restore-test.timer /etc/systemd/system/app-restore-test.timer.dist 644
+else add systemd/app-restore-test.timer /etc/systemd/system/app-restore-test.timer 644; fi
 add lib/common.sh /usr/local/lib/app-backup/common.sh 640
 add lib/locking.sh /usr/local/lib/app-backup/locking.sh 640
 add lib/backup_set.py /usr/local/lib/app-backup/backup_set.py 640
@@ -66,6 +80,7 @@ for f in "$ROOT_DIR"/etc/app-backup/*.example; do add "etc/app-backup/$(basename
 add README.md /usr/local/share/doc/app-backup/README.md 644
 add Dokumentation.md /usr/local/share/doc/app-backup/Dokumentation.md 644
 add UPDATE_ANLEITUNG.md /usr/local/share/doc/app-backup/UPDATE_ANLEITUNG.md 644
+add RESTORE_TEST.md /usr/local/share/doc/app-backup/RESTORE_TEST.md 644
 # Save the exact replaced files plus a rollback inventory; do not dereference links.
 : > "$SNAPSHOT/present.txt"; : > "$SNAPSHOT/absent.txt"
 for i in "${!TARGETS[@]}"; do
@@ -87,9 +102,10 @@ LOCKFILE=/run/app-backup.lock
 if [[ -f /etc/app-backup/app-backup.conf ]]; then source /etc/app-backup/app-backup.conf; fi
 # shellcheck disable=SC1091
 source "$here/locking.sh"
-acquire_backup_lock
+acquire_backup_lock 1
 if systemctl is-active --quiet app-backup.service; then echo 'Backup service is active'; exit 1; fi
 systemctl stop app-backup.timer
+systemctl disable --now app-restore-test.timer app-restore-test-cleanup.service || true
 while IFS= read -r target; do cp -a --remove-destination -- "$here$target" "$target"; done < "$here/present.txt"
 while IFS= read -r target; do rm -f -- "$target"; done < "$here/absent.txt"
 systemctl daemon-reload
@@ -109,6 +125,7 @@ trap on_exit EXIT
 trap 'exit 130' INT
 trap 'exit 143' TERM
 systemctl stop app-backup.timer
+if [[ $RESTORE_TIMER_ACTIVE == true ]]; then systemctl stop app-restore-test.timer; fi
 CHANGING=true
 for i in "${!TARGETS[@]}"; do
   target=${TARGETS[i]}
@@ -127,6 +144,7 @@ if [[ ${APP_BACKUP_INHERIT_LOCK:-0} != 1 ]]; then flock -u 9; fi
 if [[ $NO_TIMER == true ]]; then :
 elif [[ $ENABLE_TIMER == true ]]; then systemctl enable --now app-backup.timer
 elif [[ $TIMER_ACTIVE == true ]]; then systemctl start app-backup.timer; fi
+if [[ $NO_TIMER != true && $RESTORE_TIMER_ACTIVE == true ]]; then systemctl start app-restore-test.timer; fi
 CHANGING=false
 echo "Installed. Rollback package: $SNAPSHOT"
 echo 'Existing config and timer schedule were preserved. Compare *.dist files and review migration documentation.'