192 lines
8.5 KiB
Bash
Executable File
192 lines
8.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# A backup is published only after every source, archive and remote readback passed.
|
|
set -Eeuo pipefail
|
|
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
|
LIB_DIR="$SCRIPT_DIR/../lib/app-backup"
|
|
[[ -f $LIB_DIR/common.sh ]] || LIB_DIR="$SCRIPT_DIR/../lib"
|
|
# shellcheck source=lib/common.sh
|
|
source "$LIB_DIR/common.sh"
|
|
CONFIG_FILE=/etc/app-backup/app-backup.conf
|
|
CHECK=false
|
|
SKIP_RETENTION=false
|
|
PREPARE_SPACE=false
|
|
while (( $# )); do
|
|
case $1 in
|
|
--config) CONFIG_FILE=${2:?Missing config path}; shift 2 ;;
|
|
--check) CHECK=true; shift ;;
|
|
--prepare-space) PREPARE_SPACE=true; CHECK=true; shift ;;
|
|
--no-retention) SKIP_RETENTION=true; shift ;;
|
|
--help) echo 'Usage: app-backup.sh [--config FILE] [--check | --prepare-space] [--no-retention]'; exit 0 ;;
|
|
*) die "Unknown argument: $1" ;;
|
|
esac
|
|
done
|
|
OPERATION=backup
|
|
load_config
|
|
init_runtime
|
|
build_plan
|
|
# shellcheck source=lib/space.sh
|
|
source "$LIB_DIR/space.sh"
|
|
if [[ $CHECK == false || $PREPARE_SPACE == true ]]; then prepare_backup_space; fi
|
|
preflight
|
|
if [[ $CHECK == true ]]; then
|
|
echo 'Preflight passed. Directory metadata probed; no application backup, service stop or upload performed.'
|
|
exit 0
|
|
fi
|
|
STAGING_DIR=$(mktemp -d "$STAGING_ROOT/${RUN_ID}.XXXXXX")
|
|
mkdir -p "$STAGING_DIR"/{db,files,meta}
|
|
PARTIAL_DIR="$ARCHIVE_DIR/.partial-$RUN_ID"
|
|
mkdir -m 700 -- "$PARTIAL_DIR"
|
|
PLAN="$STAGING_DIR/meta/plan.tsv"
|
|
printf 'meta\tmetadata\n' > "$PLAN"
|
|
printf '%s\n' "$RUN_ID" > "$STAGING_DIR/meta/run-id.txt"
|
|
# The config may contain secrets or executable substitutions; never copy it into reports.
|
|
printf 'format=2\ncompressor=%s\n' "$COMPRESSOR" > "$STAGING_DIR/meta/format.txt"
|
|
BACKUP_PHASE='Dienste anhalten'
|
|
quiesce
|
|
|
|
for i in "${!DB_LABELS[@]}"; do
|
|
assert_services_quiescent
|
|
label=${DB_LABELS[i]}
|
|
out="$STAGING_DIR/db/${label#db-}.sql"
|
|
BACKUP_PHASE="DB-Dump: ${DB_NAMES[i]}"
|
|
echo "-- Dump: ${DB_NAMES[i]}"
|
|
# Defaults-extra-file must be the first option. lock-all-tables also covers MyISAM;
|
|
# this can block ALL server databases during each dump (see documentation).
|
|
"$DB_DUMP_BIN" --defaults-extra-file="${DB_CNFS[i]}" --lock-all-tables \
|
|
--quick --routines --events --triggers --hex-blob --default-character-set=utf8mb4 \
|
|
"${DB_NAMES[i]}" > "$out.partial"
|
|
assert_services_quiescent
|
|
[[ -s $out.partial ]] || die "Empty database dump: $label"
|
|
mv -- "$out.partial" "$out"
|
|
printf '%s\t%s\n' "$label" "${DB_NAMES[i]}" >> "$PLAN"
|
|
done
|
|
for i in "${!LABELS[@]}"; do
|
|
assert_services_quiescent
|
|
label=${LABELS[i]}; src=${SOURCES[i]}; root=${ROOTS[i]}
|
|
check_space "$STAGING_ROOT"
|
|
mkdir -p -- "$STAGING_DIR/$root"
|
|
exclusions "$label"
|
|
BACKUP_PHASE="Dateikopie: $label"
|
|
echo "-- Copy: $label ($src)"
|
|
# Never accept vanished-file (24) or partial-copy (23) as success.
|
|
# Parent retains both locks; rsync itself needs neither inherited descriptor.
|
|
rsync -aHAX --stats --numeric-ids "${EXCLUDES[@]}" -- "$src/" "$STAGING_DIR/$root/" 8>&- 9>&-
|
|
assert_services_quiescent
|
|
printf '%s\t%s\n' "$label" "$src" >> "$PLAN"
|
|
done
|
|
# The full consistency window includes all dumps AND copies. Compression/upload
|
|
# use this frozen private staging tree, so application downtime ends here.
|
|
assert_services_quiescent
|
|
echo '-- Copy window complete; writer guards verified'
|
|
BACKUP_PHASE='Dienste wiederherstellen'
|
|
resume_services || die 'Failed to resume original service state'
|
|
if [[ $ENABLE_MAIL_REPORT == true ]]; then
|
|
python3 "$LIB_DIR/report.py" snapshot "$LOG_DIR/report_$RUN_ID.sizes.json" "$STAGING_DIR" "$PARTIAL_DIR" || echo 'WARNING: Report sizes unavailable'
|
|
fi
|
|
|
|
while IFS=$'\t' read -r label _; do
|
|
if [[ $label == meta ]]; then root=meta
|
|
elif [[ $label == db-* ]]; then root="db/${label#db-}.sql"
|
|
else root="files/$label"; fi
|
|
check_space "$ARCHIVE_DIR"
|
|
BACKUP_PHASE="Archiv: $label"
|
|
echo "-- Archive: $label"
|
|
if [[ $COMPRESSOR == zstd ]]; then
|
|
out="$PARTIAL_DIR/$label.tar.zst"
|
|
tar --numeric-owner --acls --xattrs --selinux -C "$STAGING_DIR" -cf - -- "$root" |
|
|
zstd -T"$ZSTD_THREADS" -"$ZSTD_LEVEL" -q -c > "$out.partial"
|
|
else
|
|
out="$PARTIAL_DIR/$label.tar.gz"
|
|
tar --numeric-owner --acls --xattrs --selinux -C "$STAGING_DIR" -cf - -- "$root" |
|
|
gzip -6 -c > "$out.partial"
|
|
fi
|
|
mv -- "$out.partial" "$out"
|
|
helper tar "$out" "$root"
|
|
# Retain small metadata/plan for publication and recognized crash recovery.
|
|
if [[ $label != meta ]]; then rm -rf -- "${STAGING_DIR:?}/$root"; fi
|
|
done < "$PLAN"
|
|
BACKUP_PHASE='Lokale Archivprüfung'
|
|
helper create "$PARTIAL_DIR" "$RUN_ID" "$PLAN"
|
|
RUN_DIR="$ARCHIVE_DIR/$RUN_ID"
|
|
mv -- "$PARTIAL_DIR" "$RUN_DIR"
|
|
sync -f "$ARCHIVE_DIR"
|
|
echo "-- Complete local backup: $RUN_DIR"
|
|
|
|
if [[ $ENABLE_UPLOAD == true ]]; then
|
|
remote_run="${RCLONE_REMOTE_BASE%/}/$RUN_ID"
|
|
# Upload COMPLETE last. An interrupted upload cannot look like a valid set.
|
|
echo '-- Remote upload started (archives and manifest; completion marker withheld)'
|
|
BACKUP_PHASE='Remote-Upload'; UPLOAD_STATUS=COPY
|
|
rc copy "$RUN_DIR" "$remote_run" --exclude /COMPLETE --exclude /UPLOADED --immutable
|
|
echo '-- Remote upload finished; reading all uploaded files back for verification'
|
|
BACKUP_PHASE='Remote-Rückleseprüfung'; UPLOAD_STATUS=CHECK
|
|
remote_readback "$RUN_DIR" "$remote_run" --exclude /COMPLETE --exclude /UPLOADED
|
|
echo '-- Remote readback passed; publishing completion marker'
|
|
BACKUP_PHASE='Remote-Abschlussmarker'; UPLOAD_STATUS=MARKER
|
|
rc copyto "$RUN_DIR/COMPLETE" "$remote_run/COMPLETE" --immutable
|
|
# Command substitution strips trailing newlines, which could accept a marker
|
|
# that the strict restore parser rejects. Compare the exact returned bytes.
|
|
rc cat "$remote_run/COMPLETE" > "$STAGING_DIR/remote-COMPLETE"
|
|
python3 - "$RUN_DIR/COMPLETE" "$STAGING_DIR/remote-COMPLETE" <<'PY' || die 'Remote completion marker mismatch'
|
|
from pathlib import Path
|
|
import sys
|
|
local, remote = map(Path, sys.argv[1:])
|
|
sys.exit(0 if local.stat().st_size == remote.stat().st_size and local.read_bytes() == remote.read_bytes() else 1)
|
|
PY
|
|
date -u -Is > "$RUN_DIR/UPLOADED"
|
|
sync -f "$RUN_DIR"
|
|
UPLOAD_STATUS=VERIFIED
|
|
echo '-- Remote backup fully read back and verified'
|
|
fi
|
|
|
|
if [[ $SKIP_RETENTION == true ]]; then
|
|
BACKUP_PHASE='Abgeschlossen (Retention übersprungen)'
|
|
echo "SUCCESS: $RUN_ID (retention skipped for acceptance run)"
|
|
exit 0
|
|
fi
|
|
|
|
# Retention works on complete v2 sets only, after a new successful backup.
|
|
# Older flat archives, partial runs and unknown folders are deliberately preserved.
|
|
BACKUP_PHASE=Retention
|
|
valid="$STAGING_DIR/local-retention.txt"
|
|
: > "$valid"
|
|
for dir in "$ARCHIVE_DIR/${ARCHIVE_PREFIX}_"*; do
|
|
[[ -d $dir && ! -L $dir && -f $dir/COMPLETE ]] || continue
|
|
[[ $ENABLE_UPLOAD != true || -f $dir/UPLOADED ]] || continue
|
|
if helper metadata "$dir" >/dev/null 2>&1; then basename -- "$dir" >> "$valid"; fi
|
|
done
|
|
helper candidates "$ARCHIVE_PREFIX" "$LOCAL_RETENTION_DAYS" "$RETENTION_KEEP_LAST" < "$valid" > "$valid.delete"
|
|
while IFS= read -r name; do
|
|
[[ $name != "$RUN_ID" ]] || continue
|
|
echo "-- Local retention: $name"
|
|
rm -rf -- "${ARCHIVE_DIR:?}/$name"
|
|
done < "$valid.delete"
|
|
if [[ $ENABLE_UPLOAD == true && $ENABLE_REMOTE_RETENTION == true ]]; then
|
|
rc lsf "$RCLONE_REMOTE_BASE" --dirs-only --max-depth 1 > "$STAGING_DIR/remote-dirs.txt"
|
|
: > "$STAGING_DIR/remote-valid.txt"
|
|
mkdir "$STAGING_DIR/remote-meta"
|
|
while IFS= read -r name; do
|
|
name=${name%/}
|
|
[[ $name =~ ^${ARCHIVE_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}_[0-9a-f]{8}$ ]] || continue
|
|
# Missing markers are expected for interrupted uploads; these are NEVER purged.
|
|
if rc cat "$RCLONE_REMOTE_BASE/$name/COMPLETE" > "$STAGING_DIR/remote-meta/COMPLETE" 2>/dev/null &&
|
|
rc cat "$RCLONE_REMOTE_BASE/$name/manifest.json" > "$STAGING_DIR/remote-meta/manifest.json" 2>/dev/null &&
|
|
helper metadata "$STAGING_DIR/remote-meta" >/dev/null 2>&1; then
|
|
echo "$name" >> "$STAGING_DIR/remote-valid.txt"
|
|
fi
|
|
done < "$STAGING_DIR/remote-dirs.txt"
|
|
helper candidates "$ARCHIVE_PREFIX" "$REMOTE_RETENTION_DAYS" "$RETENTION_KEEP_LAST" < "$STAGING_DIR/remote-valid.txt" > "$STAGING_DIR/remote-delete.txt"
|
|
while IFS= read -r name; do
|
|
[[ $name != "$RUN_ID" ]] || continue
|
|
echo "-- Remote retention: $name"
|
|
rc purge "$RCLONE_REMOTE_BASE/$name"
|
|
done < "$STAGING_DIR/remote-delete.txt"
|
|
fi
|
|
# Per-run filenames need age-based cleanup, not rotating an empty successor file.
|
|
for operation in backup restore report; do
|
|
find "$LOG_DIR" -maxdepth 1 -type f \( -name "${operation}_${ARCHIVE_PREFIX}_*.log" -o -name "${operation}_${ARCHIVE_PREFIX}_*.eml" \) \
|
|
-mtime "+${LOG_RETENTION_DAYS}" ! -name "$(basename "$LOG_FILE")" -delete
|
|
done
|
|
BACKUP_PHASE=Abgeschlossen
|
|
echo "SUCCESS: $RUN_ID"
|