Initial version of JR IT backup of internal backend.

2026-02-02 08:37:51 +01:00
commit 144de4f393
10 changed files with 969 additions and 0 deletions
--- a/usr-local-sbin/app-backup.sh
+++ b/usr-local-sbin/app-backup.sh
@@ -0,0 +1,473 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 027
+
+# ---------- Logging ----------
+LOG_DIR="/var/log/app-backup"
+mkdir -p "$LOG_DIR"
+ts="$(date '+%Y-%m-%d_%H-%M-%S')"
+LOG_FILE="${LOG_DIR}/app-backup_${ts}.log"
+
+# Log to file + journald
+exec > >(tee -a "$LOG_FILE" | systemd-cat -t app-backup -p info) 2>&1
+
+# ---------- Config ----------
+CONFIG_FILE="/etc/app-backup/app-backup.conf"
+if [[ -r "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+else
+  echo "ERROR: Config not found/readable: $CONFIG_FILE"
+  exit 2
+fi
+
+# ---------- Defaults (if not set in conf) ----------
+: "${WORKDIR:=/var/backups/app-backup}"
+: "${STAGING_ROOT:=${WORKDIR}/staging}"
+: "${ARCHIVE_DIR:=${WORKDIR}/archives}"
+: "${LOCAL_RETENTION_DAYS:=7}"
+
+: "${COMPRESSOR:=zstd}"          # zstd|gzip
+: "${RCLONE_REMOTE:=onedrive:Sicherung}"
+: "${RCLONE_BIN:=rclone}"
+: "${RCLONE_RETRIES:=6}"
+: "${RCLONE_LOW_LEVEL_RETRIES:=20}"
+: "${RCLONE_RETRIES_SLEEP:=30s}"
+: "${RCLONE_STATS:=1m}"
+: "${RCLONE_BWLIMIT:=0}"         # "0" = no limit
+: "${REMOTE_RETENTION_DAYS:=30}"
+: "${ENABLE_REMOTE_RETENTION:=true}"
+
+# Disk-space safety: minimum free space required on the filesystem holding WORKDIR
+: "${MIN_FREE_GB:=10}"
+
+# Process niceness
+: "${NICE_LEVEL:=10}"
+: "${IONICE_CLASS:=2}"           # best-effort
+: "${IONICE_LEVEL:=6}"
+
+# Mail reporting (local postfix via sendmail)
+: "${ENABLE_MAIL_REPORT:=true}"
+: "${MAIL_TO:=johannes}"              # local mailbox
+: "${MAIL_FROM:=app-backup@$(hostname -f 2>/dev/null || hostname)}"
+: "${MAIL_SUBJECT_PREFIX:=[app-backup]}"
+: "${MAIL_INCLUDE_LOG_TAIL_LINES:=200}"
+
+# ---------- State for report ----------
+START_EPOCH="$(date +%s)"
+STATUS="SUCCESS"
+ERROR_SUMMARY=""
+RCLONE_STATUS="SKIPPED"
+RCLONE_OUTPUT_FILE=""
+ARCHIVE_FILE=""
+SIZES_FILE=""
+DELETED_LOCAL_COUNT=0
+DELETED_LOCAL_BYTES=0
+DELETED_LOCAL_LIST_FILE=""
+
+# ---------- Helpers ----------
+die() { echo "ERROR: $*"; exit 1; }
+have() { command -v "$1" >/dev/null 2>&1; }
+
+human_bytes() {
+  local b="${1:-0}"
+  if have numfmt; then numfmt --to=iec-i --suffix=B "$b"; else echo "${b}B"; fi
+}
+
+bytes_of_path() {
+  local p="$1"
+  if [[ -e "$p" ]]; then
+    du -sb "$p" 2>/dev/null | awk '{print $1}' || du -sB1 "$p" | awk '{print $1}'
+  else
+    echo 0
+  fi
+}
+
+free_bytes_workdir_fs() {
+  # available bytes on filesystem that contains WORKDIR
+  df -PB1 "$WORKDIR" | awk 'NR==2{print $4}'
+}
+
+cleanup_old_local_archives() {
+  mkdir -p "$ARCHIVE_DIR"
+  local min_age_days="$LOCAL_RETENTION_DAYS"
+  local list_file="${WORKDIR}/deleted_local_archives_${ts}.txt"
+  DELETED_LOCAL_LIST_FILE="$list_file"
+  : > "$list_file"
+
+  # collect files older than retention
+  # shellcheck disable=SC2016
+  while IFS= read -r -d '' f; do
+    # best-effort size
+    local sz
+    sz="$(stat -c '%s' "$f" 2>/dev/null || echo 0)"
+    DELETED_LOCAL_BYTES=$((DELETED_LOCAL_BYTES + sz))
+    DELETED_LOCAL_COUNT=$((DELETED_LOCAL_COUNT + 1))
+    echo "$f" >> "$list_file"
+  done < <(find "$ARCHIVE_DIR" -type f -name 'appbackup_*.tar.*' -mtime "+${min_age_days}" -print0 2>/dev/null || true)
+
+  if [[ "$DELETED_LOCAL_COUNT" -gt 0 ]]; then
+    echo "-- Local retention: deleting ${DELETED_LOCAL_COUNT} archive(s) older than ${LOCAL_RETENTION_DAYS} day(s) from ${ARCHIVE_DIR}"
+    # delete using the collected list to ensure count/bytes match what we report
+    while IFS= read -r f; do
+      rm -f -- "$f" || true
+    done < "$list_file"
+    echo "-- Local retention: freed approx $(human_bytes "$DELETED_LOCAL_BYTES")"
+  else
+    echo "-- Local retention: nothing to delete (keep last ${LOCAL_RETENTION_DAYS} day(s))"
+  fi
+}
+
+ensure_min_free_space() {
+  local min_bytes=$((MIN_FREE_GB * 1024 * 1024 * 1024))
+  local avail
+  avail="$(free_bytes_workdir_fs)"
+  echo "-- Free space on WORKDIR filesystem: $(human_bytes "$avail") (min required: ${MIN_FREE_GB}GiB)"
+  if [[ "$avail" -lt "$min_bytes" ]]; then
+    die "Not enough free space on WORKDIR filesystem (need >= ${MIN_FREE_GB}GiB). Aborting to prevent disk full."
+  fi
+}
+
+send_report_mail() {
+  [[ "${ENABLE_MAIL_REPORT}" == "true" ]] || return 0
+  local SENDMAIL_BIN="/usr/sbin/sendmail"
+  if [[ ! -x "$SENDMAIL_BIN" ]]; then
+    echo "WARN: sendmail binary not found/executable at $SENDMAIL_BIN - cannot send report mail"
+    return 0
+  fi
+
+  local end_epoch now duration subject host
+  end_epoch="$(date +%s)"
+  now="$(date -Is)"
+  duration="$((end_epoch - START_EPOCH))"
+  host="$(hostname -f 2>/dev/null || hostname)"
+  subject="${MAIL_SUBJECT_PREFIX} ${STATUS} ${host} ${ts}"
+
+  local recs
+  recs=$(
+    cat <<'EOF'
+Empfehlungen (mehr Resilienz):
+- Regelmäßig Restore testen (Stichprobe): DB-Import + Dateien entpacken + App-Start prüfen.
+- Verschlüsselung: Archiv zusätzlich clientseitig verschlüsseln (z.B. age/gpg) bevor Upload.
+- Immutable/Versioned Backup-Ziel (wenn möglich): Schutz vor Ransomware/Löschung.
+- Monitoring/Alerting: systemd unit failure => Benachrichtigung.
+- Snapshots (LVM/Btrfs/ZFS): Bei großen Datenmengen besser als rsync; reduziert Downtime für Nextcloud.
+EOF
+  )
+
+  {
+    echo "From: ${MAIL_FROM}"
+    echo "To: ${MAIL_TO}"
+    echo "Subject: ${subject}"
+    echo "Date: $(date -R)"
+    echo "MIME-Version: 1.0"
+    echo "Content-Type: text/plain; charset=UTF-8"
+    echo
+    echo "Backup Report"
+    echo "============="
+    echo "Zeit: ${now}"
+    echo "Host: ${host}"
+    echo "Status: ${STATUS}"
+    [[ -n "${ERROR_SUMMARY}" ]] && echo "Fehler: ${ERROR_SUMMARY}"
+    echo "Dauer: ${duration}s"
+    echo
+    echo "Konfiguration"
+    echo "------------"
+    echo "Config: ${CONFIG_FILE}"
+    echo "Log: ${LOG_FILE}"
+    echo "Workdir: ${WORKDIR}"
+    echo "Archive dir: ${ARCHIVE_DIR}"
+    echo "Kompression: ${COMPRESSOR}"
+    echo
+    echo "Disk / Retention"
+    echo "---------------"
+    echo "Min. freier Speicher: ${MIN_FREE_GB}GiB"
+    echo "Lokale Aufbewahrung: ${LOCAL_RETENTION_DAYS} Tage"
+    echo "Gelöschte lokale Backups: ${DELETED_LOCAL_COUNT} (ca. $(human_bytes "${DELETED_LOCAL_BYTES}"))"
+    if [[ -n "${DELETED_LOCAL_LIST_FILE}" && -s "${DELETED_LOCAL_LIST_FILE}" ]]; then
+      echo "Gelöschte Dateien (erste 20):"
+      head -n 20 "${DELETED_LOCAL_LIST_FILE}" || true
+    fi
+    if [[ "${ENABLE_REMOTE_RETENTION}" == "true" ]]; then
+      echo "Remote Aufbewahrung: ${REMOTE_RETENTION_DAYS} Tage (rclone delete --min-age)"
+    else
+      echo "Remote Aufbewahrung: deaktiviert"
+    fi
+    echo
+    echo "Was wurde gesichert?"
+    echo "-------------------"
+    [[ "${ENABLE_WORDPRESS}" == "true" ]] && echo "- WordPress Dateien: ${WP_DIR}" || true
+    [[ "${ENABLE_DB_DUMPS}" == "true" && -n "${WP_DB_NAME:-}" ]] && echo "- WordPress DB: ${WP_DB_NAME}" || true
+    [[ "${ENABLE_NEXTCLOUD}" == "true" ]] && echo "- Nextcloud Dateien: ${NC_DIR}" || true
+    [[ "${ENABLE_NEXTCLOUD_DATA}" == "true" ]] && echo "- Nextcloud Data: ${NC_DATA_DIR}" || true
+    [[ "${ENABLE_DB_DUMPS}" == "true" && -n "${NC_DB_NAME:-}" ]] && echo "- Nextcloud DB: ${NC_DB_NAME}" || true
+    [[ "${ENABLE_MAIL}" == "true" ]] && echo "- Mail: ${MAIL_DIR:-<unset>} + ${POSTFIX_DIR:-<unset>} + ${DOVECOT_DIR:-<unset>}" || true
+    echo
+    echo "Größen"
+    echo "------"
+    if [[ -n "${SIZES_FILE}" && -f "${SIZES_FILE}" ]]; then
+      cat "${SIZES_FILE}"
+    else
+      echo "(keine Größeninfos verfügbar)"
+    fi
+    echo
+    echo "Upload (rclone)"
+    echo "--------------"
+    echo "Remote: ${RCLONE_REMOTE}"
+    echo "Upload Status: ${RCLONE_STATUS}"
+    if [[ -n "${RCLONE_OUTPUT_FILE}" && -f "${RCLONE_OUTPUT_FILE}" ]]; then
+      echo
+      echo "rclone Output (Tail):"
+      tail -n 50 "${RCLONE_OUTPUT_FILE}" || true
+    fi
+    echo
+    echo "${recs}"
+    echo
+    echo "Log-Auszug (Tail)"
+    echo "-----------------"
+    tail -n "${MAIL_INCLUDE_LOG_TAIL_LINES}" "${LOG_FILE}" || true
+  } | "$SENDMAIL_BIN" -t || echo "WARN: sending mail failed (postfix queue may still accept it)"
+}
+
+cleanup_staging() {
+  if [[ -n "${STAGING_DIR:-}" && -d "${STAGING_DIR:-}" ]]; then
+    rm -rf "${STAGING_DIR:?}"
+  fi
+}
+
+# Nextcloud maintenance-mode safety trap
+NC_MAINTENANCE_ON=false
+nc_maintenance_off() {
+  if [[ "${NC_MAINTENANCE_ON}" == "true" ]]; then
+    echo "-- Nextcloud maintenance mode OFF (trap)..."
+    sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --off || true
+    NC_MAINTENANCE_ON=false
+  fi
+}
+
+on_error() {
+  local exit_code=$?
+  STATUS="FAIL"
+  ERROR_SUMMARY="Exit code ${exit_code} (see log)"
+  return 0
+}
+
+on_exit() {
+  local exit_code=$?
+  send_report_mail
+  nc_maintenance_off
+  cleanup_staging
+  exit "${exit_code}"
+}
+
+trap on_error ERR
+trap on_exit EXIT
+
+# ---------- Start ----------
+echo "== app-backup start: ${ts} =="
+echo "-- Host: $(hostname -f 2>/dev/null || hostname)"
+echo "-- Config: ${CONFIG_FILE}"
+echo "-- Log: ${LOG_FILE}"
+
+# ---------- Preconditions ----------
+[[ $EUID -eq 0 ]] || die "Must run as root."
+
+mkdir -p "$WORKDIR" "$ARCHIVE_DIR" "$STAGING_ROOT" "$LOG_DIR"
+
+# ---------- Locking (prevents parallel runs) ----------
+LOCKFILE="/run/app-backup.lock"
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+  STATUS="FAIL"
+  ERROR_SUMMARY="Another backup already running (lock: $LOCKFILE)"
+  exit 3
+fi
+
+# ---------- Tools ----------
+for t in tar rsync flock df find stat; do
+  have "$t" || die "Missing required tool: $t"
+done
+
+if [[ "$COMPRESSOR" == "zstd" ]]; then
+  have zstd || die "COMPRESSOR=zstd but zstd is missing"
+elif [[ "$COMPRESSOR" == "gzip" ]]; then
+  have gzip || die "COMPRESSOR=gzip but gzip is missing"
+else
+  die "Unsupported COMPRESSOR=$COMPRESSOR (use zstd or gzip)"
+fi
+
+if [[ "${ENABLE_DB_DUMPS}" == "true" ]]; then
+  have mysqldump || die "ENABLE_DB_DUMPS=true but mysqldump missing"
+fi
+
+have "$RCLONE_BIN" || die "rclone not installed (missing: $RCLONE_BIN)"
+
+# ---------- Disk safety: cleanup + free-space checks ----------
+cleanup_old_local_archives
+ensure_min_free_space
+
+# ---------- Staging ----------
+STAGING_DIR="${STAGING_ROOT}/run_${ts}"
+mkdir -p "$STAGING_DIR"/{db,files,meta}
+
+echo "$(date -Is)" > "$STAGING_DIR/meta/created_at.txt"
+echo "$(hostname -f 2>/dev/null || hostname)" > "$STAGING_DIR/meta/hostname.txt"
+echo "${ts}" > "$STAGING_DIR/meta/timestamp.txt"
+
+# ---------- DB Dumps ----------
+if [[ "${ENABLE_DB_DUMPS}" == "true" ]]; then
+  echo "-- DB dumps enabled"
+
+  if [[ -n "${WP_DB_NAME:-}" ]]; then
+    [[ -r "${WP_DB_CNF}" ]] || die "WP_DB_CNF not readable: ${WP_DB_CNF}"
+    echo "-- Dump WordPress DB: ${WP_DB_NAME}"
+    mysqldump --defaults-extra-file="${WP_DB_CNF}" \
+      --single-transaction --routines --triggers --hex-blob \
+      "${WP_DB_NAME}" > "$STAGING_DIR/db/wordpress_${ts}.sql"
+  fi
+
+  if [[ -n "${NC_DB_NAME:-}" ]]; then
+    [[ -r "${NC_DB_CNF}" ]] || die "NC_DB_CNF not readable: ${NC_DB_CNF}"
+    echo "-- Dump Nextcloud DB: ${NC_DB_NAME}"
+
+    if [[ "${ENABLE_NEXTCLOUD_MAINTENANCE}" == "true" ]]; then
+      echo "-- Nextcloud maintenance mode ON..."
+      sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --on
+      NC_MAINTENANCE_ON=true
+    fi
+
+    mysqldump --defaults-extra-file="${NC_DB_CNF}" \
+      --single-transaction --routines --triggers --hex-blob \
+      "${NC_DB_NAME}" > "$STAGING_DIR/db/nextcloud_${ts}.sql"
+
+    if [[ "${ENABLE_NEXTCLOUD_MAINTENANCE}" == "true" ]]; then
+      echo "-- Nextcloud maintenance mode OFF..."
+      sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --off
+      NC_MAINTENANCE_ON=false
+    fi
+  fi
+else
+  echo "-- DB dumps disabled"
+fi
+
+# ---------- File Copies (rsync into staging for consistency) ----------
+echo "-- Collecting files via rsync..."
+
+rsync_dir() {
+  local src="$1"
+  local dst="$2"
+  [[ -d "$src" ]] || die "Source directory missing: $src"
+  mkdir -p "$dst"
+  rsync -aHAX --numeric-ids --delete --info=stats2 "$src"/ "$dst"/
+}
+
+if [[ "${ENABLE_WORDPRESS}" == "true" ]]; then
+  echo "-- WordPress files: ${WP_DIR}"
+  rsync_dir "${WP_DIR}" "$STAGING_DIR/files/wordpress"
+fi
+
+if [[ "${ENABLE_NEXTCLOUD}" == "true" ]]; then
+  echo "-- Nextcloud files: ${NC_DIR}"
+  rsync_dir "${NC_DIR}" "$STAGING_DIR/files/nextcloud"
+
+  if [[ "${ENABLE_NEXTCLOUD_DATA}" == "true" ]]; then
+    echo "-- Nextcloud data: ${NC_DATA_DIR}"
+    rsync_dir "${NC_DATA_DIR}" "$STAGING_DIR/files/nextcloud-data"
+  fi
+fi
+
+if [[ "${ENABLE_MAIL}" == "true" ]]; then
+  echo "-- Mail files..."
+  [[ -n "${MAIL_DIR:-}" && -d "${MAIL_DIR}" ]] && rsync_dir "${MAIL_DIR}" "$STAGING_DIR/files/mail" || true
+  [[ -n "${POSTFIX_DIR:-}" && -d "${POSTFIX_DIR}" ]] && rsync -aHAX "${POSTFIX_DIR}/" "$STAGING_DIR/files/postfix/" || true
+  [[ -n "${DOVECOT_DIR:-}" && -d "${DOVECOT_DIR}" ]] && rsync -aHAX "${DOVECOT_DIR}/" "$STAGING_DIR/files/dovecot/" || true
+fi
+
+# ---------- Size summary ----------
+SIZES_FILE="${STAGING_DIR}/meta/sizes.txt"
+{
+  echo "WordPress staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/wordpress")")"
+  echo "Nextcloud staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/nextcloud")")"
+  echo "Nextcloud-data staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/nextcloud-data")")"
+  echo "Mail staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/mail")")"
+  echo "DB dumps staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/db")")"
+  echo "Staging total: $(human_bytes "$(bytes_of_path "$STAGING_DIR")")"
+} > "$SIZES_FILE" || true
+
+# ---------- Disk safety: check again after staging ----------
+ensure_min_free_space
+
+# ---------- Archive ----------
+archive_base="appbackup_${ts}"
+tar_file="${ARCHIVE_DIR}/${archive_base}.tar"
+
+echo "-- Creating tar: ${tar_file}"
+(
+  cd "$STAGING_DIR"
+  tar --numeric-owner --xattrs --acls -cf "$tar_file" .
+)
+
+if [[ "$COMPRESSOR" == "zstd" ]]; then
+  ARCHIVE_FILE="${tar_file}.zst"
+  echo "-- Compressing (zstd): ${ARCHIVE_FILE}"
+  ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" \
+    zstd -T0 -19 --rm "$tar_file"
+  echo "-- Testing zstd integrity..."
+  zstd -t "$ARCHIVE_FILE"
+elif [[ "$COMPRESSOR" == "gzip" ]]; then
+  ARCHIVE_FILE="${tar_file}.gz"
+  echo "-- Compressing (gzip): ${ARCHIVE_FILE}"
+  ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" \
+    gzip -9 "$tar_file"
+  echo "-- Testing gzip integrity..."
+  gzip -t "$ARCHIVE_FILE"
+fi
+
+echo "-- Archive ready: ${ARCHIVE_FILE}"
+echo "-- Archive size: $(du -h "$ARCHIVE_FILE" | awk '{print $1}')"
+
+# ---------- rclone remote health-check ----------
+echo "-- rclone remote check: ${RCLONE_REMOTE}"
+"$RCLONE_BIN" lsf "${RCLONE_REMOTE}" --max-depth 1 >/dev/null 2>&1 || die "Remote not reachable: ${RCLONE_REMOTE}"
+
+# ---------- Upload ----------
+RCLONE_OUTPUT_FILE="${LOG_DIR}/rclone_${ts}.log"
+echo "-- Uploading via rclone (output: ${RCLONE_OUTPUT_FILE})..."
+RCLONE_STATUS="RUNNING"
+
+RCLONE_ARGS=(
+  "copy" "$ARCHIVE_FILE" "${RCLONE_REMOTE}"
+  "--checksum"
+  "--retries" "${RCLONE_RETRIES}"
+  "--low-level-retries" "${RCLONE_LOW_LEVEL_RETRIES}"
+  "--retries-sleep" "${RCLONE_RETRIES_SLEEP}"
+  "--stats" "${RCLONE_STATS}"
+  "--stats-one-line"
+  "--log-level" "INFO"
+  "--transfers" "4"
+  "--checkers" "8"
+)
+
+if [[ "${RCLONE_BWLIMIT}" != "0" ]]; then
+  RCLONE_ARGS+=("--bwlimit" "${RCLONE_BWLIMIT}")
+fi
+
+if ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" \
+  "$RCLONE_BIN" "${RCLONE_ARGS[@]}" | tee -a "$RCLONE_OUTPUT_FILE"
+then
+  RCLONE_STATUS="OK"
+else
+  RCLONE_STATUS="FAIL"
+  die "Upload failed (see ${RCLONE_OUTPUT_FILE})"
+fi
+
+# ---------- Remote retention ----------
+if [[ "${ENABLE_REMOTE_RETENTION}" == "true" ]]; then
+  echo "-- Remote retention: delete older than ${REMOTE_RETENTION_DAYS}d"
+  "$RCLONE_BIN" delete "${RCLONE_REMOTE}" --min-age "${REMOTE_RETENTION_DAYS}d" --log-level INFO || true
+fi
+
+# ---------- Local retention (again, to enforce after new archive) ----------
+cleanup_old_local_archives
+
+echo "== app-backup done: ${ts} =="
--- a/usr-local-sbin/app-restore.sh
+++ b/usr-local-sbin/app-restore.sh
@@ -0,0 +1,243 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 027
+
+# ---------------- Logging ----------------
+LOG_DIR="/var/log/app-backup"
+mkdir -p "$LOG_DIR"
+ts="$(date '+%Y-%m-%d_%H-%M-%S')"
+LOG_FILE="${LOG_DIR}/app-restore_${ts}.log"
+exec > >(tee -a "$LOG_FILE" | systemd-cat -t app-restore -p info) 2>&1
+
+# ---------------- Config ----------------
+CONFIG_FILE="/etc/app-backup/app-backup.conf"
+if [[ -r "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+else
+  echo "ERROR: Config not found/readable: $CONFIG_FILE"
+  exit 2
+fi
+
+: "${WORKDIR:=/var/backups/app-backup}"
+: "${STAGING_ROOT:=${WORKDIR}/restore-staging}"
+: "${RCLONE_REMOTE:=onedrive:Sicherung}"
+: "${RCLONE_BIN:=rclone}"
+: "${NC_OCC_USER:=apache}"
+
+ARCHIVE_PATH=""
+REMOTE_NAME=""
+DRY_RUN=false
+FORCE=false
+
+DO_WORDPRESS=true
+DO_NEXTCLOUD=true
+DO_NEXTCLOUD_DATA=true
+DO_MAIL=true
+DO_DB=true
+
+usage() {
+  cat <<EOF
+Usage:
+  app-restore.sh --archive /path/to/appbackup_....tar.(zst|gz)
+  app-restore.sh --remote-file appbackup_....tar.(zst|gz)
+Options:
+  --archive <path>        Local archive path.
+  --remote-file <name>    Download that file name from rclone remote (${RCLONE_REMOTE}) into ${WORKDIR}/restore-downloads/.
+  --dry-run               No writes (rsync --dry-run; DB import skipped; services not restarted).
+  --force                 Skip some safety checks.
+  --only <list>           Comma-separated: wordpress,nextcloud,nextcloud-data,mail,db
+  --skip <list>           Comma-separated: wordpress,nextcloud,nextcloud-data,mail,db
+  -h|--help               Help.
+EOF
+}
+
+set_all_false() { DO_WORDPRESS=false; DO_NEXTCLOUD=false; DO_NEXTCLOUD_DATA=false; DO_MAIL=false; DO_DB=false; }
+parse_list_flags() {
+  local mode="$1" list="$2"
+  IFS=',' read -r -a items <<< "$list"
+  for it in "${items[@]}"; do
+    it="$(echo "$it" | tr '[:upper:]' '[:lower:]' | xargs)"
+    case "$it" in
+      wordpress)       [[ "$mode" == "only" ]] && DO_WORDPRESS=true || DO_WORDPRESS=false ;;
+      nextcloud)       [[ "$mode" == "only" ]] && DO_NEXTCLOUD=true || DO_NEXTCLOUD=false ;;
+      nextcloud-data)  [[ "$mode" == "only" ]] && DO_NEXTCLOUD_DATA=true || DO_NEXTCLOUD_DATA=false ;;
+      mail)            [[ "$mode" == "only" ]] && DO_MAIL=true || DO_MAIL=false ;;
+      db)              [[ "$mode" == "only" ]] && DO_DB=true || DO_DB=false ;;
+      "" ) : ;;
+      *) echo "WARN: unknown component: $it" ;;
+    esac
+  done
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --archive) ARCHIVE_PATH="${2:-}"; shift 2 ;;
+    --remote-file) REMOTE_NAME="${2:-}"; shift 2 ;;
+    --dry-run) DRY_RUN=true; shift ;;
+    --force) FORCE=true; shift ;;
+    --only) set_all_false; parse_list_flags "only" "${2:-}"; shift 2 ;;
+    --skip) parse_list_flags "skip" "${2:-}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "ERROR: unknown arg: $1"; usage; exit 2 ;;
+  esac
+done
+
+die(){ echo "ERROR: $*"; exit 1; }
+have(){ command -v "$1" >/dev/null 2>&1; }
+service_is_active(){ systemctl is-active --quiet "$1"; }
+
+STOPPED_SERVICES=()
+stop_service_if_active() {
+  local s="$1"
+  if service_is_active "$s"; then
+    echo "-- Stopping service: $s"
+    [[ "$DRY_RUN" == "true" ]] || systemctl stop "$s"
+    STOPPED_SERVICES+=("$s")
+  fi
+}
+
+start_stopped_services() {
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "-- DRY RUN: not starting services"
+    return 0
+  fi
+  for s in "${STOPPED_SERVICES[@]}"; do
+    echo "-- Starting service: $s"
+    systemctl start "$s" || echo "WARN: failed to start $s"
+  done
+}
+
+integrity_check() {
+  local f="$1"
+  if [[ "$f" == *.zst ]]; then
+    have zstd || die "zstd missing but archive is .zst"
+    zstd -t "$f"
+  elif [[ "$f" == *.gz ]]; then
+    have gzip || die "gzip missing but archive is .gz"
+    gzip -t "$f"
+  else
+    die "Unsupported archive extension: $f"
+  fi
+}
+
+extract_archive() {
+  local f="$1" dest="$2"
+  mkdir -p "$dest"
+  if [[ "$f" == *.zst ]]; then
+    tar --use-compress-program=unzstd -xf "$f" -C "$dest"
+  else
+    tar -xzf "$f" -C "$dest"
+  fi
+}
+
+rsync_restore_dir() {
+  local src="$1" dst="$2"
+  [[ -d "$src" ]] || die "Staging source missing: $src"
+  mkdir -p "$dst"
+  local args=(-aHAX --numeric-ids --delete)
+  [[ "$DRY_RUN" == "true" ]] && args+=(--dry-run)
+  echo "-- Restoring: $src => $dst"
+  rsync "${args[@]}" "$src"/ "$dst"/
+}
+
+mysql_import() {
+  local cnf="$1" db="$2" sql="$3"
+  [[ -r "$cnf" ]] || die "DB cnf not readable: $cnf"
+  [[ -r "$sql" ]] || die "SQL dump not readable: $sql"
+  have mysql || die "mysql client missing"
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "-- DRY RUN: skipping DB import for $db"
+    return 0
+  fi
+  mysql --defaults-extra-file="$cnf" -e "CREATE DATABASE IF NOT EXISTS \`${db}\`;"
+  mysql --defaults-extra-file="$cnf" "$db" < "$sql"
+}
+
+NC_MAINTENANCE_ON=false
+nc_maintenance_on() {
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "-- DRY RUN: not toggling Nextcloud maintenance mode"
+    return 0
+  fi
+  if [[ -f "${NC_DIR}/occ" ]]; then
+    sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --on
+    NC_MAINTENANCE_ON=true
+  fi
+}
+nc_maintenance_off() {
+  if [[ "${NC_MAINTENANCE_ON}" == "true" ]]; then
+    sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --off || true
+    NC_MAINTENANCE_ON=false
+  fi
+}
+
+echo "== app-restore start: ${ts} =="
+echo "-- Log: ${LOG_FILE}"
+[[ $EUID -eq 0 ]] || die "Must run as root."
+
+LOCKFILE="/run/app-restore.lock"
+exec 9>"$LOCKFILE"
+flock -n 9 || die "Another restore is already running (lock: $LOCKFILE)"
+
+DOWNLOAD_DIR="${WORKDIR}/restore-downloads"
+mkdir -p "$DOWNLOAD_DIR"
+
+if [[ -n "$REMOTE_NAME" ]]; then
+  have "$RCLONE_BIN" || die "rclone required for --remote-file"
+  ARCHIVE_PATH="${DOWNLOAD_DIR}/${REMOTE_NAME}"
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "-- DRY RUN: would download ${RCLONE_REMOTE}/${REMOTE_NAME} to ${ARCHIVE_PATH}"
+  else
+    "$RCLONE_BIN" copy "${RCLONE_REMOTE}/${REMOTE_NAME}" "$DOWNLOAD_DIR" --log-level INFO
+  fi
+fi
+
+[[ -n "$ARCHIVE_PATH" ]] || { usage; die "Provide --archive or --remote-file"; }
+[[ -f "$ARCHIVE_PATH" ]] || die "Archive not found: $ARCHIVE_PATH"
+
+echo "-- Selected archive: $ARCHIVE_PATH"
+integrity_check "$ARCHIVE_PATH"
+
+STAGING_DIR="${STAGING_ROOT}/run_${ts}"
+trap 'nc_maintenance_off; start_stopped_services; rm -rf "${STAGING_DIR:?}"' EXIT
+extract_archive "$ARCHIVE_PATH" "$STAGING_DIR"
+
+[[ -d "$STAGING_DIR/files" ]] || die "Invalid archive: missing 'files' dir"
+
+# stop services to prevent writes
+if systemctl list-unit-files | grep -q '^httpd\.service'; then stop_service_if_active httpd; fi
+if [[ "$DO_MAIL" == "true" ]]; then
+  if systemctl list-unit-files | grep -q '^postfix\.service'; then stop_service_if_active postfix; fi
+  if systemctl list-unit-files | grep -q '^dovecot\.service'; then stop_service_if_active dovecot; fi
+fi
+
+# nextcloud maintenance for restore
+if [[ "$DO_NEXTCLOUD" == "true" || "$DO_NEXTCLOUD_DATA" == "true" || "$DO_DB" == "true" ]]; then
+  nc_maintenance_on
+fi
+
+# restore files
+[[ "$DO_WORDPRESS" == "true" && -d "$STAGING_DIR/files/wordpress" ]] && rsync_restore_dir "$STAGING_DIR/files/wordpress" "$WP_DIR" || true
+[[ "$DO_NEXTCLOUD" == "true" && -d "$STAGING_DIR/files/nextcloud" ]] && rsync_restore_dir "$STAGING_DIR/files/nextcloud" "$NC_DIR" || true
+[[ "$DO_NEXTCLOUD_DATA" == "true" && -d "$STAGING_DIR/files/nextcloud-data" ]] && rsync_restore_dir "$STAGING_DIR/files/nextcloud-data" "$NC_DATA_DIR" || true
+
+if [[ "$DO_MAIL" == "true" ]]; then
+  [[ -d "$STAGING_DIR/files/mail" && -n "${MAIL_DIR:-}" ]] && rsync_restore_dir "$STAGING_DIR/files/mail" "$MAIL_DIR" || true
+  [[ -d "$STAGING_DIR/files/postfix" && -n "${POSTFIX_DIR:-}" ]] && rsync_restore_dir "$STAGING_DIR/files/postfix" "$POSTFIX_DIR" || true
+  [[ -d "$STAGING_DIR/files/dovecot" && -n "${DOVECOT_DIR:-}" ]] && rsync_restore_dir "$STAGING_DIR/files/dovecot" "$DOVECOT_DIR" || true
+fi
+
+# restore DBs
+if [[ "$DO_DB" == "true" && "${ENABLE_DB_DUMPS:-false}" == "true" ]]; then
+  WP_SQL="$(ls -1 "$STAGING_DIR/db"/wordpress_*.sql 2>/dev/null | tail -n 1 || true)"
+  NC_SQL="$(ls -1 "$STAGING_DIR/db"/nextcloud_*.sql 2>/dev/null | tail -n 1 || true)"
+  [[ "$DO_WORDPRESS" == "true" && -n "${WP_DB_NAME:-}" && -n "$WP_SQL" ]] && mysql_import "$WP_DB_CNF" "$WP_DB_NAME" "$WP_SQL" || true
+  [[ ( "$DO_NEXTCLOUD" == "true" || "$DO_NEXTCLOUD_DATA" == "true" ) && -n "${NC_DB_NAME:-}" && -n "$NC_SQL" ]] && mysql_import "$NC_DB_CNF" "$NC_DB_NAME" "$NC_SQL" || true
+fi
+
+nc_maintenance_off
+start_stopped_services
+
+echo "== app-restore done: ${ts} =="
+echo "-- Log: ${LOG_FILE}"