#!/usr/bin/env bash set -Eeuo pipefail umask 027 # ---------- Logging ---------- LOG_DIR="/var/log/app-backup" mkdir -p "$LOG_DIR" ts="$(date '+%Y-%m-%d_%H-%M-%S')" LOG_FILE="${LOG_DIR}/app-backup_${ts}.log" # Log to file + journald exec > >(tee -a "$LOG_FILE" | systemd-cat -t app-backup -p info) 2>&1 # ---------- Config ---------- CONFIG_FILE="/etc/app-backup/app-backup.conf" if [[ -r "$CONFIG_FILE" ]]; then # shellcheck disable=SC1090 source "$CONFIG_FILE" else echo "ERROR: Config not found/readable: $CONFIG_FILE" exit 2 fi # ---------- Defaults (if not set in conf) ---------- : "${WORKDIR:=/var/backups/app-backup}" : "${STAGING_ROOT:=${WORKDIR}/staging}" : "${ARCHIVE_DIR:=${WORKDIR}/archives}" : "${LOCAL_RETENTION_DAYS:=7}" : "${COMPRESSOR:=zstd}" # zstd|gzip : "${RCLONE_REMOTE:=onedrive:Sicherung}" : "${RCLONE_BIN:=rclone}" : "${RCLONE_RETRIES:=6}" : "${RCLONE_LOW_LEVEL_RETRIES:=20}" : "${RCLONE_RETRIES_SLEEP:=30s}" : "${RCLONE_STATS:=1m}" : "${RCLONE_BWLIMIT:=0}" # "0" = no limit : "${REMOTE_RETENTION_DAYS:=30}" : "${ENABLE_REMOTE_RETENTION:=true}" # Disk-space safety: minimum free space required on the filesystem holding WORKDIR : "${MIN_FREE_GB:=10}" # Process niceness : "${NICE_LEVEL:=10}" : "${IONICE_CLASS:=2}" # best-effort : "${IONICE_LEVEL:=6}" # Mail reporting (local postfix via sendmail) : "${ENABLE_MAIL_REPORT:=true}" : "${MAIL_TO:=johannes}" # local mailbox : "${MAIL_FROM:=app-backup@$(hostname -f 2>/dev/null || hostname)}" : "${MAIL_SUBJECT_PREFIX:=[app-backup]}" : "${MAIL_INCLUDE_LOG_TAIL_LINES:=200}" # ---------- State for report ---------- START_EPOCH="$(date +%s)" STATUS="SUCCESS" ERROR_SUMMARY="" RCLONE_STATUS="SKIPPED" RCLONE_OUTPUT_FILE="" ARCHIVE_FILE="" SIZES_FILE="" DELETED_LOCAL_COUNT=0 DELETED_LOCAL_BYTES=0 DELETED_LOCAL_LIST_FILE="" # ---------- Helpers ---------- die() { echo "ERROR: $*"; exit 1; } have() { command -v "$1" >/dev/null 2>&1; } human_bytes() { local b="${1:-0}" if have numfmt; then numfmt --to=iec-i --suffix=B "$b"; else echo "${b}B"; fi } bytes_of_path() { local p="$1" if [[ -e "$p" ]]; then du -sb "$p" 2>/dev/null | awk '{print $1}' || du -sB1 "$p" | awk '{print $1}' else echo 0 fi } free_bytes_workdir_fs() { # available bytes on filesystem that contains WORKDIR df -PB1 "$WORKDIR" | awk 'NR==2{print $4}' } cleanup_old_local_archives() { mkdir -p "$ARCHIVE_DIR" local min_age_days="$LOCAL_RETENTION_DAYS" local list_file="${WORKDIR}/deleted_local_archives_${ts}.txt" DELETED_LOCAL_LIST_FILE="$list_file" : > "$list_file" # collect files older than retention # shellcheck disable=SC2016 while IFS= read -r -d '' f; do # best-effort size local sz sz="$(stat -c '%s' "$f" 2>/dev/null || echo 0)" DELETED_LOCAL_BYTES=$((DELETED_LOCAL_BYTES + sz)) DELETED_LOCAL_COUNT=$((DELETED_LOCAL_COUNT + 1)) echo "$f" >> "$list_file" done < <(find "$ARCHIVE_DIR" -type f -name 'appbackup_*.tar.*' -mtime "+${min_age_days}" -print0 2>/dev/null || true) if [[ "$DELETED_LOCAL_COUNT" -gt 0 ]]; then echo "-- Local retention: deleting ${DELETED_LOCAL_COUNT} archive(s) older than ${LOCAL_RETENTION_DAYS} day(s) from ${ARCHIVE_DIR}" # delete using the collected list to ensure count/bytes match what we report while IFS= read -r f; do rm -f -- "$f" || true done < "$list_file" echo "-- Local retention: freed approx $(human_bytes "$DELETED_LOCAL_BYTES")" else echo "-- Local retention: nothing to delete (keep last ${LOCAL_RETENTION_DAYS} day(s))" fi } ensure_min_free_space() { local min_bytes=$((MIN_FREE_GB * 1024 * 1024 * 1024)) local avail avail="$(free_bytes_workdir_fs)" echo "-- Free space on WORKDIR filesystem: $(human_bytes "$avail") (min required: ${MIN_FREE_GB}GiB)" if [[ "$avail" -lt "$min_bytes" ]]; then die "Not enough free space on WORKDIR filesystem (need >= ${MIN_FREE_GB}GiB). Aborting to prevent disk full." fi } send_report_mail() { [[ "${ENABLE_MAIL_REPORT}" == "true" ]] || return 0 local SENDMAIL_BIN="/usr/sbin/sendmail" if [[ ! -x "$SENDMAIL_BIN" ]]; then echo "WARN: sendmail binary not found/executable at $SENDMAIL_BIN - cannot send report mail" return 0 fi local end_epoch now duration subject host end_epoch="$(date +%s)" now="$(date -Is)" duration="$((end_epoch - START_EPOCH))" host="$(hostname -f 2>/dev/null || hostname)" subject="${MAIL_SUBJECT_PREFIX} ${STATUS} ${host} ${ts}" local recs recs=$( cat <<'EOF' Empfehlungen (mehr Resilienz): - Regelmäßig Restore testen (Stichprobe): DB-Import + Dateien entpacken + App-Start prüfen. - Verschlüsselung: Archiv zusätzlich clientseitig verschlüsseln (z.B. age/gpg) bevor Upload. - Immutable/Versioned Backup-Ziel (wenn möglich): Schutz vor Ransomware/Löschung. - Monitoring/Alerting: systemd unit failure => Benachrichtigung. - Snapshots (LVM/Btrfs/ZFS): Bei großen Datenmengen besser als rsync; reduziert Downtime für Nextcloud. EOF ) { echo "From: ${MAIL_FROM}" echo "To: ${MAIL_TO}" echo "Subject: ${subject}" echo "Date: $(date -R)" echo "MIME-Version: 1.0" echo "Content-Type: text/plain; charset=UTF-8" echo echo "Backup Report" echo "=============" echo "Zeit: ${now}" echo "Host: ${host}" echo "Status: ${STATUS}" [[ -n "${ERROR_SUMMARY}" ]] && echo "Fehler: ${ERROR_SUMMARY}" echo "Dauer: ${duration}s" echo echo "Konfiguration" echo "------------" echo "Config: ${CONFIG_FILE}" echo "Log: ${LOG_FILE}" echo "Workdir: ${WORKDIR}" echo "Archive dir: ${ARCHIVE_DIR}" echo "Kompression: ${COMPRESSOR}" echo echo "Disk / Retention" echo "---------------" echo "Min. freier Speicher: ${MIN_FREE_GB}GiB" echo "Lokale Aufbewahrung: ${LOCAL_RETENTION_DAYS} Tage" echo "Gelöschte lokale Backups: ${DELETED_LOCAL_COUNT} (ca. $(human_bytes "${DELETED_LOCAL_BYTES}"))" if [[ -n "${DELETED_LOCAL_LIST_FILE}" && -s "${DELETED_LOCAL_LIST_FILE}" ]]; then echo "Gelöschte Dateien (erste 20):" head -n 20 "${DELETED_LOCAL_LIST_FILE}" || true fi if [[ "${ENABLE_REMOTE_RETENTION}" == "true" ]]; then echo "Remote Aufbewahrung: ${REMOTE_RETENTION_DAYS} Tage (rclone delete --min-age)" else echo "Remote Aufbewahrung: deaktiviert" fi echo echo "Was wurde gesichert?" echo "-------------------" [[ "${ENABLE_WORDPRESS}" == "true" ]] && echo "- WordPress Dateien: ${WP_DIR}" || true [[ "${ENABLE_DB_DUMPS}" == "true" && -n "${WP_DB_NAME:-}" ]] && echo "- WordPress DB: ${WP_DB_NAME}" || true [[ "${ENABLE_NEXTCLOUD}" == "true" ]] && echo "- Nextcloud Dateien: ${NC_DIR}" || true [[ "${ENABLE_NEXTCLOUD_DATA}" == "true" ]] && echo "- Nextcloud Data: ${NC_DATA_DIR}" || true [[ "${ENABLE_DB_DUMPS}" == "true" && -n "${NC_DB_NAME:-}" ]] && echo "- Nextcloud DB: ${NC_DB_NAME}" || true [[ "${ENABLE_MAIL}" == "true" ]] && echo "- Mail: ${MAIL_DIR:-} + ${POSTFIX_DIR:-} + ${DOVECOT_DIR:-}" || true echo echo "Größen" echo "------" if [[ -n "${SIZES_FILE}" && -f "${SIZES_FILE}" ]]; then cat "${SIZES_FILE}" else echo "(keine Größeninfos verfügbar)" fi echo echo "Upload (rclone)" echo "--------------" echo "Remote: ${RCLONE_REMOTE}" echo "Upload Status: ${RCLONE_STATUS}" if [[ -n "${RCLONE_OUTPUT_FILE}" && -f "${RCLONE_OUTPUT_FILE}" ]]; then echo echo "rclone Output (Tail):" tail -n 50 "${RCLONE_OUTPUT_FILE}" || true fi echo echo "${recs}" echo echo "Log-Auszug (Tail)" echo "-----------------" tail -n "${MAIL_INCLUDE_LOG_TAIL_LINES}" "${LOG_FILE}" || true } | "$SENDMAIL_BIN" -t || echo "WARN: sending mail failed (postfix queue may still accept it)" } cleanup_staging() { if [[ -n "${STAGING_DIR:-}" && -d "${STAGING_DIR:-}" ]]; then rm -rf "${STAGING_DIR:?}" fi } # Nextcloud maintenance-mode safety trap NC_MAINTENANCE_ON=false nc_maintenance_off() { if [[ "${NC_MAINTENANCE_ON}" == "true" ]]; then echo "-- Nextcloud maintenance mode OFF (trap)..." sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --off || true NC_MAINTENANCE_ON=false fi } on_error() { local exit_code=$? STATUS="FAIL" ERROR_SUMMARY="Exit code ${exit_code} (see log)" return 0 } on_exit() { local exit_code=$? send_report_mail nc_maintenance_off cleanup_staging exit "${exit_code}" } trap on_error ERR trap on_exit EXIT # ---------- Start ---------- echo "== app-backup start: ${ts} ==" echo "-- Host: $(hostname -f 2>/dev/null || hostname)" echo "-- Config: ${CONFIG_FILE}" echo "-- Log: ${LOG_FILE}" # ---------- Preconditions ---------- [[ $EUID -eq 0 ]] || die "Must run as root." mkdir -p "$WORKDIR" "$ARCHIVE_DIR" "$STAGING_ROOT" "$LOG_DIR" # ---------- Locking (prevents parallel runs) ---------- LOCKFILE="/run/app-backup.lock" exec 9>"$LOCKFILE" if ! flock -n 9; then STATUS="FAIL" ERROR_SUMMARY="Another backup already running (lock: $LOCKFILE)" exit 3 fi # ---------- Tools ---------- for t in tar rsync flock df find stat; do have "$t" || die "Missing required tool: $t" done if [[ "$COMPRESSOR" == "zstd" ]]; then have zstd || die "COMPRESSOR=zstd but zstd is missing" elif [[ "$COMPRESSOR" == "gzip" ]]; then have gzip || die "COMPRESSOR=gzip but gzip is missing" else die "Unsupported COMPRESSOR=$COMPRESSOR (use zstd or gzip)" fi if [[ "${ENABLE_DB_DUMPS}" == "true" ]]; then have mysqldump || die "ENABLE_DB_DUMPS=true but mysqldump missing" fi have "$RCLONE_BIN" || die "rclone not installed (missing: $RCLONE_BIN)" # ---------- Disk safety: cleanup + free-space checks ---------- cleanup_old_local_archives ensure_min_free_space # ---------- Staging ---------- STAGING_DIR="${STAGING_ROOT}/run_${ts}" mkdir -p "$STAGING_DIR"/{db,files,meta} echo "$(date -Is)" > "$STAGING_DIR/meta/created_at.txt" echo "$(hostname -f 2>/dev/null || hostname)" > "$STAGING_DIR/meta/hostname.txt" echo "${ts}" > "$STAGING_DIR/meta/timestamp.txt" # ---------- DB Dumps ---------- if [[ "${ENABLE_DB_DUMPS}" == "true" ]]; then echo "-- DB dumps enabled" if [[ -n "${WP_DB_NAME:-}" ]]; then [[ -r "${WP_DB_CNF}" ]] || die "WP_DB_CNF not readable: ${WP_DB_CNF}" echo "-- Dump WordPress DB: ${WP_DB_NAME}" mysqldump --defaults-extra-file="${WP_DB_CNF}" \ --single-transaction --routines --triggers --hex-blob \ "${WP_DB_NAME}" > "$STAGING_DIR/db/wordpress_${ts}.sql" fi if [[ -n "${NC_DB_NAME:-}" ]]; then [[ -r "${NC_DB_CNF}" ]] || die "NC_DB_CNF not readable: ${NC_DB_CNF}" echo "-- Dump Nextcloud DB: ${NC_DB_NAME}" if [[ "${ENABLE_NEXTCLOUD_MAINTENANCE}" == "true" ]]; then echo "-- Nextcloud maintenance mode ON..." sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --on NC_MAINTENANCE_ON=true fi mysqldump --defaults-extra-file="${NC_DB_CNF}" \ --single-transaction --routines --triggers --hex-blob \ "${NC_DB_NAME}" > "$STAGING_DIR/db/nextcloud_${ts}.sql" if [[ "${ENABLE_NEXTCLOUD_MAINTENANCE}" == "true" ]]; then echo "-- Nextcloud maintenance mode OFF..." sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --off NC_MAINTENANCE_ON=false fi fi else echo "-- DB dumps disabled" fi # ---------- File Copies (rsync into staging for consistency) ---------- echo "-- Collecting files via rsync..." rsync_dir() { local src="$1" local dst="$2" [[ -d "$src" ]] || die "Source directory missing: $src" mkdir -p "$dst" rsync -aHAX --numeric-ids --delete --info=stats2 "$src"/ "$dst"/ } if [[ "${ENABLE_WORDPRESS}" == "true" ]]; then echo "-- WordPress files: ${WP_DIR}" rsync_dir "${WP_DIR}" "$STAGING_DIR/files/wordpress" fi if [[ "${ENABLE_NEXTCLOUD}" == "true" ]]; then echo "-- Nextcloud files: ${NC_DIR}" rsync_dir "${NC_DIR}" "$STAGING_DIR/files/nextcloud" if [[ "${ENABLE_NEXTCLOUD_DATA}" == "true" ]]; then echo "-- Nextcloud data: ${NC_DATA_DIR}" rsync_dir "${NC_DATA_DIR}" "$STAGING_DIR/files/nextcloud-data" fi fi if [[ "${ENABLE_MAIL}" == "true" ]]; then echo "-- Mail files..." [[ -n "${MAIL_DIR:-}" && -d "${MAIL_DIR}" ]] && rsync_dir "${MAIL_DIR}" "$STAGING_DIR/files/mail" || true [[ -n "${POSTFIX_DIR:-}" && -d "${POSTFIX_DIR}" ]] && rsync -aHAX "${POSTFIX_DIR}/" "$STAGING_DIR/files/postfix/" || true [[ -n "${DOVECOT_DIR:-}" && -d "${DOVECOT_DIR}" ]] && rsync -aHAX "${DOVECOT_DIR}/" "$STAGING_DIR/files/dovecot/" || true fi # ---------- Size summary ---------- SIZES_FILE="${STAGING_DIR}/meta/sizes.txt" { echo "WordPress staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/wordpress")")" echo "Nextcloud staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/nextcloud")")" echo "Nextcloud-data staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/nextcloud-data")")" echo "Mail staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/mail")")" echo "DB dumps staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/db")")" echo "Staging total: $(human_bytes "$(bytes_of_path "$STAGING_DIR")")" } > "$SIZES_FILE" || true # ---------- Disk safety: check again after staging ---------- ensure_min_free_space # ---------- Archive ---------- archive_base="appbackup_${ts}" tar_file="${ARCHIVE_DIR}/${archive_base}.tar" echo "-- Creating tar: ${tar_file}" ( cd "$STAGING_DIR" tar --numeric-owner --xattrs --acls -cf "$tar_file" . ) if [[ "$COMPRESSOR" == "zstd" ]]; then ARCHIVE_FILE="${tar_file}.zst" echo "-- Compressing (zstd): ${ARCHIVE_FILE}" ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" \ zstd -T0 -19 --rm "$tar_file" echo "-- Testing zstd integrity..." zstd -t "$ARCHIVE_FILE" elif [[ "$COMPRESSOR" == "gzip" ]]; then ARCHIVE_FILE="${tar_file}.gz" echo "-- Compressing (gzip): ${ARCHIVE_FILE}" ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" \ gzip -9 "$tar_file" echo "-- Testing gzip integrity..." gzip -t "$ARCHIVE_FILE" fi echo "-- Archive ready: ${ARCHIVE_FILE}" echo "-- Archive size: $(du -h "$ARCHIVE_FILE" | awk '{print $1}')" # ---------- rclone remote health-check ---------- echo "-- rclone remote check: ${RCLONE_REMOTE}" "$RCLONE_BIN" lsf "${RCLONE_REMOTE}" --max-depth 1 >/dev/null 2>&1 || die "Remote not reachable: ${RCLONE_REMOTE}" # ---------- Upload ---------- RCLONE_OUTPUT_FILE="${LOG_DIR}/rclone_${ts}.log" echo "-- Uploading via rclone (output: ${RCLONE_OUTPUT_FILE})..." RCLONE_STATUS="RUNNING" RCLONE_ARGS=( "copy" "$ARCHIVE_FILE" "${RCLONE_REMOTE}" "--checksum" "--retries" "${RCLONE_RETRIES}" "--low-level-retries" "${RCLONE_LOW_LEVEL_RETRIES}" "--retries-sleep" "${RCLONE_RETRIES_SLEEP}" "--stats" "${RCLONE_STATS}" "--stats-one-line" "--log-level" "INFO" "--transfers" "4" "--checkers" "8" ) if [[ "${RCLONE_BWLIMIT}" != "0" ]]; then RCLONE_ARGS+=("--bwlimit" "${RCLONE_BWLIMIT}") fi if ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" \ "$RCLONE_BIN" "${RCLONE_ARGS[@]}" | tee -a "$RCLONE_OUTPUT_FILE" then RCLONE_STATUS="OK" else RCLONE_STATUS="FAIL" die "Upload failed (see ${RCLONE_OUTPUT_FILE})" fi # ---------- Remote retention ---------- if [[ "${ENABLE_REMOTE_RETENTION}" == "true" ]]; then echo "-- Remote retention: delete older than ${REMOTE_RETENTION_DAYS}d" "$RCLONE_BIN" delete "${RCLONE_REMOTE}" --min-age "${REMOTE_RETENTION_DAYS}d" --log-level INFO || true fi # ---------- Local retention (again, to enforce after new archive) ---------- cleanup_old_local_archives echo "== app-backup done: ${ts} =="