Prepare MariaDB restore tests and recover backup disk space safely

This commit is contained in:
admin committed 2026-09-12 20:36:18 +02:00
1 parent f0c761213d
commit b0416461bc
23 files changed
+918 -129

No files matched your search

+16 -3
View File
@@ -40,7 +40,7 @@ load_config() {
: "${MAIL_HOSTNAME:=}"
: "${MAIL_SENDMAIL_BIN:=/usr/sbin/sendmail}" "${MAIL_SEND_TIMEOUT:=30}"
: "${ENABLE_RESTORE_TEST:=false}" "${RESTORE_TEST_DB_IMAGE:=}"
: "${RESTORE_TEST_MAX_WORK_GB:=4}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
: "${RESTORE_TEST_MAX_WORK_GB:=0}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
: "${RESTORE_TEST_MAX_AGE_HOURS:=72}" "${RESTORE_TEST_TIMEOUT:=39600}"
: "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
: "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
@@ -71,6 +71,7 @@ load_config() {
[[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
done
(( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
(( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
(( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
[[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
[[ $COMPRESSOR == zstd || $COMPRESSOR == gzip ]] || die "COMPRESSOR must be zstd or gzip"
@@ -487,6 +488,7 @@ resume_services() {
}
finish() {
local code=$?
local recovery_ok=true
trap - EXIT INT TERM
set +e
if [[ $OPERATION == restore && $MUTATION_STARTED == true && $code != 0 ]]; then
@@ -495,7 +497,7 @@ finish() {
echo "ERROR: Partial restore. Service recovery requires manual review: ${STOPPED_SERVICES[*]}"
echo "ERROR: Keep Nextcloud in maintenance; review $LOG_FILE before restarting."
else
resume_services || code=1
if ! resume_services; then code=1; recovery_ok=false; fi
fi
local report_sizes="$LOG_DIR/report_$RUN_ID.sizes.json"
if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
@@ -503,7 +505,18 @@ finish() {
python3 "$LIB_DIR/report.py" snapshot "$report_sizes" "${STAGING_DIR:-}" "${RUN_DIR:-${PARTIAL_DIR:-}}" ||
echo 'WARNING: Report sizes unavailable'
fi
if [[ $code == 0 && -n ${STAGING_DIR:-} ]]; then rm -rf -- "$STAGING_DIR" || code=1; fi
if [[ $OPERATION == backup && $recovery_ok == true && -n ${STAGING_DIR:-} && ! -e $WORKDIR/recovery-$RUN_ID.json ]]; then
# A failed backup scratch copy is disposable once writer recovery succeeded.
# Never delete a locally complete set awaiting upload or a completion marker.
if [[ -n ${PARTIAL_DIR:-} && -d $PARTIAL_DIR && ! -e $PARTIAL_DIR/COMPLETE ]]; then
python3 "$LIB_DIR/space.py" remove "$PARTIAL_DIR" || code=1
fi
if [[ ! -d ${PARTIAL_DIR:-/nonexistent} ]]; then
python3 "$LIB_DIR/space.py" remove "$STAGING_DIR" && STAGING_DIR='' || code=1
fi
elif [[ ( $code == 0 || ( $OPERATION == restore && ${MODE:-} == verify ) ) && -n ${STAGING_DIR:-} ]]; then
rm -rf -- "$STAGING_DIR" && STAGING_DIR='' || code=1
fi
if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi
# Close the logger's writers and wait for EOF so the mail tail includes the
# final recovery/status messages. Reporting itself uses the original stdout.