Add monthly offsite restore tests and automatic cleanup

lib/common.sh

@@ -39,6 +39,9 @@ load_config() {
   : "${MAIL_FROM:=}" "${MAIL_SUBJECT_PREFIX:=[app-backup]}" "${MAIL_INCLUDE_LOG_TAIL_LINES:=200}"
   : "${MAIL_HOSTNAME:=}"
   : "${MAIL_SENDMAIL_BIN:=/usr/sbin/sendmail}" "${MAIL_SEND_TIMEOUT:=30}"
+  : "${ENABLE_RESTORE_TEST:=false}" "${RESTORE_TEST_DB_IMAGE:=}"
+  : "${RESTORE_TEST_MAX_WORK_GB:=4}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
+  : "${RESTORE_TEST_MAX_AGE_HOURS:=72}" "${RESTORE_TEST_TIMEOUT:=39600}"
   : "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
   : "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
   : "${ENABLE_NEXTCLOUD_MAINTENANCE:=true}" "${NC_OCC_USER:=apache}"
@@ -61,7 +64,7 @@ load_config() {
   fi
   for v in ENABLE_UPLOAD ENABLE_REMOTE_RETENTION ENABLE_MAIL_REPORT ENABLE_WORDPRESS ENABLE_NEXTCLOUD \
       ENABLE_NEXTCLOUD_DATA ENABLE_GITEA ENABLE_MAIL ENABLE_MAIL_STORE ENABLE_DB_DUMPS ENABLE_NEXTCLOUD_MAINTENANCE \
-      ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE; do
+      ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE ENABLE_RESTORE_TEST; do
     [[ ${!v} == true || ${!v} == false ]] || die "$v must be true or false"
   done
   for v in LOG_RETENTION_DAYS MIN_FREE_GB LOCAL_RETENTION_DAYS REMOTE_RETENTION_DAYS RETENTION_KEEP_LAST ZSTD_LEVEL ZSTD_THREADS; do
@@ -110,7 +113,7 @@ init_runtime() {
   guard_workspaces
   # shellcheck source=lib/locking.sh
   source "$LIB_DIR/locking.sh"
-  acquire_backup_lock
+  acquire_backup_lock 1
   for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
     [[ $p == /* && $p != / && $p != *$'\n'* && $p != *$'\t'* ]] || die "Unsafe working path: $p"
     [[ ! -L $p ]] || die "Symlink working directory refused: $p"