Prepare MariaDB restore tests and recover backup disk space safely
This commit is contained in:
1 parent
f0c761213d
commit
b0416461bc
23 files changed
+918
-129
No files matched your search
+16
-3
@@ -40,7 +40,7 @@ load_config() {
|
||||
: "${MAIL_HOSTNAME:=}"
|
||||
: "${MAIL_SENDMAIL_BIN:=/usr/sbin/sendmail}" "${MAIL_SEND_TIMEOUT:=30}"
|
||||
: "${ENABLE_RESTORE_TEST:=false}" "${RESTORE_TEST_DB_IMAGE:=}"
|
||||
: "${RESTORE_TEST_MAX_WORK_GB:=4}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
|
||||
: "${RESTORE_TEST_MAX_WORK_GB:=0}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
|
||||
: "${RESTORE_TEST_MAX_AGE_HOURS:=72}" "${RESTORE_TEST_TIMEOUT:=39600}"
|
||||
: "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
|
||||
: "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
|
||||
@@ -71,6 +71,7 @@ load_config() {
|
||||
[[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
|
||||
done
|
||||
(( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
|
||||
(( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
|
||||
(( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
|
||||
[[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
|
||||
[[ $COMPRESSOR == zstd || $COMPRESSOR == gzip ]] || die "COMPRESSOR must be zstd or gzip"
|
||||
@@ -487,6 +488,7 @@ resume_services() {
|
||||
}
|
||||
finish() {
|
||||
local code=$?
|
||||
local recovery_ok=true
|
||||
trap - EXIT INT TERM
|
||||
set +e
|
||||
if [[ $OPERATION == restore && $MUTATION_STARTED == true && $code != 0 ]]; then
|
||||
@@ -495,7 +497,7 @@ finish() {
|
||||
echo "ERROR: Partial restore. Service recovery requires manual review: ${STOPPED_SERVICES[*]}"
|
||||
echo "ERROR: Keep Nextcloud in maintenance; review $LOG_FILE before restarting."
|
||||
else
|
||||
resume_services || code=1
|
||||
if ! resume_services; then code=1; recovery_ok=false; fi
|
||||
fi
|
||||
local report_sizes="$LOG_DIR/report_$RUN_ID.sizes.json"
|
||||
if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
|
||||
@@ -503,7 +505,18 @@ finish() {
|
||||
python3 "$LIB_DIR/report.py" snapshot "$report_sizes" "${STAGING_DIR:-}" "${RUN_DIR:-${PARTIAL_DIR:-}}" ||
|
||||
echo 'WARNING: Report sizes unavailable'
|
||||
fi
|
||||
if [[ $code == 0 && -n ${STAGING_DIR:-} ]]; then rm -rf -- "$STAGING_DIR" || code=1; fi
|
||||
if [[ $OPERATION == backup && $recovery_ok == true && -n ${STAGING_DIR:-} && ! -e $WORKDIR/recovery-$RUN_ID.json ]]; then
|
||||
# A failed backup scratch copy is disposable once writer recovery succeeded.
|
||||
# Never delete a locally complete set awaiting upload or a completion marker.
|
||||
if [[ -n ${PARTIAL_DIR:-} && -d $PARTIAL_DIR && ! -e $PARTIAL_DIR/COMPLETE ]]; then
|
||||
python3 "$LIB_DIR/space.py" remove "$PARTIAL_DIR" || code=1
|
||||
fi
|
||||
if [[ ! -d ${PARTIAL_DIR:-/nonexistent} ]]; then
|
||||
python3 "$LIB_DIR/space.py" remove "$STAGING_DIR" && STAGING_DIR='' || code=1
|
||||
fi
|
||||
elif [[ ( $code == 0 || ( $OPERATION == restore && ${MODE:-} == verify ) ) && -n ${STAGING_DIR:-} ]]; then
|
||||
rm -rf -- "$STAGING_DIR" && STAGING_DIR='' || code=1
|
||||
fi
|
||||
if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi
|
||||
# Close the logger's writers and wait for EOF so the mail tail includes the
|
||||
# final recovery/status messages. Reporting itself uses the original stdout.
|
||||
|
||||
Reference in new issue
Block a user