Add monthly offsite restore tests and automatic cleanup

systemd/app-restore-test-cleanup.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Clean interrupted app-backup restore tests after boot
+After=local-fs.target docker.service
+Wants=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash /usr/local/sbin/app-restore-test.sh --cleanup
+SELinuxContext=system_u:system_r:unconfined_service_t:s0
+TimeoutStartSec=5min
+UMask=0077
+Restart=on-failure
+RestartSec=15min
+
+[Install]
+WantedBy=multi-user.target

systemd/app-restore-test.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=Monthly offsite restore test with automatic scratch cleanup
+Wants=network-online.target docker.service
+After=network-online.target docker.service
+OnFailure=app-restore-test-cleanup.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash /usr/local/sbin/app-restore-test.sh
+ExecStopPost=/bin/bash /usr/local/sbin/app-restore-test.sh --cleanup
+SELinuxContext=system_u:system_r:unconfined_service_t:s0
+TimeoutStartSec=12h
+TimeoutStopSec=5min
+KillMode=control-group
+# A busy shared lock is retried later; a failed restore remains a visible failure.
+Restart=no
+RestartForceExitStatus=75
+RestartSec=1h
+UMask=0077
+Nice=15
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+Environment=LANG=C
+Environment=LC_ALL=C
+StandardOutput=journal
+StandardError=journal

systemd/app-restore-test.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Test the latest offsite backup on the first day of each month
+
+[Timer]
+OnCalendar=*-*-01 06:00:00
+RandomizedDelaySec=15m
+Persistent=true
+
+[Install]
+WantedBy=timers.target