#!/usr/bin/env bash
# Shared safety rules. Configuration is trusted root-owned Bash, never archive input.
set -Eeuo pipefail
shopt -s inherit_errexit
umask 077
export LC_ALL=C

die() { echo "ERROR: $*" >&2; exit 1; }
need() { command -v "$1" >/dev/null || die "Missing tool: $1"; }
helper() { python3 "$LIB_DIR/backup_set.py" "$@"; }

secure_file() {
  [[ -f "$1" && ! -L "$1" ]] || die "Expected regular file: $1"
  [[ $(stat -c %u "$1") == 0 ]] || die "File must belong to root: $1"
  (( (8#$(stat -c %a "$1") & 8#022) == 0 )) || die "File writable by group/others: $1"
}

load_config() {
  [[ $EUID == 0 ]] || die "Run as root."
  secure_file "$CONFIG_FILE"
  # shellcheck disable=SC1090
  source "$CONFIG_FILE"
  : "${WORKDIR:=/var/backups/app-backup}"
  : "${STAGING_ROOT:=$WORKDIR/staging}" "${ARCHIVE_DIR:=$WORKDIR/archives}"
  : "${RESTORE_ROOT:=$WORKDIR/restore}" "${LOG_DIR:=/var/log/app-backup}"
  : "${LOCKFILE:=/run/app-backup.lock}" "${ARCHIVE_PREFIX:=appbackup}"
  : "${COMPRESSOR:=zstd}" "${ZSTD_LEVEL:=6}" "${ZSTD_THREADS:=2}"
  : "${LOG_RETENTION_DAYS:=30}"
  : "${MIN_FREE_GB:=10}" "${LOCAL_RETENTION_DAYS:=7}" "${REMOTE_RETENTION_DAYS:=30}"
  : "${RETENTION_KEEP_LAST:=3}" "${ENABLE_REMOTE_RETENTION:=false}"
  : "${SPACE_KEEP_LOCAL:=1}"
  : "${ENABLE_UPLOAD:=true}" "${RCLONE_BIN:=rclone}"
  : "${RCLONE_REMOTE_BASE:=}" "${RCLONE_CONFIG:=/root/.config/rclone/rclone.conf}"
  : "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}"
  : "${RCLONE_RETRIES_SLEEP:=30s}" "${RCLONE_TIMEOUT:=1h}" "${RCLONE_CONTIMEOUT:=30s}"
  : "${RCLONE_STATS_INTERVAL:=1m}"
  : "${RCLONE_CHECK_ATTEMPTS:=3}" "${RCLONE_CHECK_RETRY_DELAY:=30}"
  : "${RCLONE_CHECK_TIMEOUT:=300}" "${RCLONE_CHECK_DEADLINE:=3600}" "${RCLONE_CHECK_LOW_LEVEL_RETRIES:=3}"
  : "${RCLONE_TRANSFERS:=2}" "${RCLONE_CHECKERS:=4}" "${RCLONE_ONEDRIVE_CHUNK_SIZE:=80Mi}"
  : "${RCLONE_BWLIMIT:=0}" "${ENABLE_MAIL_REPORT:=true}" "${MAIL_TO:=johannes}"
  # Keep the old report settings; an empty sender uses app-backup@<FQDN>.
  : "${MAIL_FROM:=}" "${MAIL_SUBJECT_PREFIX:=[app-backup]}" "${MAIL_INCLUDE_LOG_TAIL_LINES:=200}"
  : "${MAIL_HOSTNAME:=}"
  : "${MAIL_SENDMAIL_BIN:=/usr/sbin/sendmail}" "${MAIL_SEND_TIMEOUT:=30}"
  : "${ENABLE_RESTORE_TEST:=false}" "${RESTORE_TEST_DB_IMAGE:=}"
  : "${RESTORE_TEST_MAX_WORK_GB:=0}" "${RESTORE_TEST_MIN_FREE_GB:=2}"
  : "${RESTORE_TEST_MAX_AGE_HOURS:=72}" "${RESTORE_TEST_TIMEOUT:=39600}"
  : "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
  : "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
  : "${ENABLE_NEXTCLOUD_MAINTENANCE:=true}" "${NC_OCC_USER:=apache}"
  : "${ENABLE_GITEA_SERVICE_STOP:=true}" "${GITEA_SERVICE_NAME:=gitea}"
  : "${GITEA_DATA_DIR:=/var/lib/gitea/data}" "${GITEA_ETC_DIR:=/etc/gitea}"
  : "${ENABLE_HTTPD_STOP:=true}" "${HTTPD_SERVICE_NAME:=httpd}"
  : "${ENABLE_PHPFPM_STOP:=true}" "${PHPFPM_SERVICE_NAME:=php-fpm}"
  : "${MAIL_BACKUP_MODE:=stop}" "${ENABLE_MAIL_HOME_DIRS:=true}" "${ENABLE_MAIL_STORE:=true}"
  : "${MAIL_DIR:=/var/vmail}" "${POSTFIX_DIR:=/etc/postfix}" "${DOVECOT_DIR:=/etc/dovecot}"
  : "${RESTORE_STRICT_DELETE:=false}"
  # Preserve old space-separated lists; arrays additionally support spaces in paths.
  if ! declare -p MAIL_HOME_DIRS &>/dev/null; then MAIL_HOME_DIRS=(/home/johannes /home/luciana); fi
  if [[ $(declare -p MAIL_HOME_DIRS) != 'declare -a '* ]]; then
    read -r -a MAIL_HOME_DIRS <<< "${MAIL_HOME_DIRS[0]}"
  fi
  if ! declare -p MAIL_HOME_EXCLUDES &>/dev/null; then
    MAIL_HOME_EXCLUDES=(OneDrive/ .cache/ .ccache/ .local/share/Trash/)
  elif [[ $(declare -p MAIL_HOME_EXCLUDES) != 'declare -a '* ]]; then
    read -r -a MAIL_HOME_EXCLUDES <<< "${MAIL_HOME_EXCLUDES[0]}"
  fi
  for v in ENABLE_UPLOAD ENABLE_REMOTE_RETENTION ENABLE_MAIL_REPORT ENABLE_WORDPRESS ENABLE_NEXTCLOUD \
      ENABLE_NEXTCLOUD_DATA ENABLE_GITEA ENABLE_MAIL ENABLE_MAIL_STORE ENABLE_DB_DUMPS ENABLE_NEXTCLOUD_MAINTENANCE \
      ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE ENABLE_RESTORE_TEST; do
    [[ ${!v} == true || ${!v} == false ]] || die "$v must be true or false"
  done
  for v in LOG_RETENTION_DAYS MIN_FREE_GB LOCAL_RETENTION_DAYS REMOTE_RETENTION_DAYS RETENTION_KEEP_LAST ZSTD_LEVEL ZSTD_THREADS; do
    [[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
  done
  (( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
  [[ $SPACE_KEEP_LOCAL =~ ^[1-9][0-9]{0,5}$ ]] || die 'SPACE_KEEP_LOCAL must be an integer between 1 and 999999'
  # Seconds/counts only: never permit disabled deadlines or unbounded retries.
  for v in RCLONE_CHECK_ATTEMPTS RCLONE_CHECK_LOW_LEVEL_RETRIES RCLONE_CHECK_TIMEOUT RCLONE_CHECK_DEADLINE RCLONE_CHECK_RETRY_DELAY; do
    [[ ${!v} =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "$v must be a bounded integer"
  done
  (( RCLONE_CHECK_ATTEMPTS >= 1 && RCLONE_CHECK_ATTEMPTS <= 10 && RCLONE_CHECK_LOW_LEVEL_RETRIES >= 1 && RCLONE_CHECK_LOW_LEVEL_RETRIES <= 20 )) || die 'Invalid remote check retry limits'
  (( RCLONE_CHECK_TIMEOUT >= 1 && RCLONE_CHECK_TIMEOUT <= 3600 && RCLONE_CHECK_DEADLINE >= 1 && RCLONE_CHECK_DEADLINE <= 86400 && RCLONE_CHECK_RETRY_DELAY <= 3600 )) || die 'Invalid remote check time limits'
  (( MIN_FREE_GB <= 1000000 )) || die 'MIN_FREE_GB must not exceed 1000000'
  (( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
  [[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
  [[ $COMPRESSOR == zstd || $COMPRESSOR == gzip ]] || die "COMPRESSOR must be zstd or gzip"
  [[ $MAIL_BACKUP_MODE == stop || $MAIL_BACKUP_MODE == live ]] || die "MAIL_BACKUP_MODE must be stop or live"
  # A homes-only setup must still contain mailbox data, not just server config.
  if [[ $ENABLE_MAIL == true && $ENABLE_MAIL_STORE == false ]]; then
    [[ $ENABLE_MAIL_HOME_DIRS == true && ${#MAIL_HOME_DIRS[@]} -gt 0 ]] || die 'Mail without a central store requires enabled, nonempty MAIL_HOME_DIRS'
  fi
  if [[ -z ${DB_DUMP_BIN:-} ]]; then DB_DUMP_BIN=$(command -v mariadb-dump || command -v mysqldump || true); fi
  if [[ -z ${DB_CLIENT_BIN:-} ]]; then DB_CLIENT_BIN=$(command -v mariadb || command -v mysql || true); fi
}

guard_workspaces() {
  local p src resolved other
  local sources=("${EXTRA_PATHS[@]}")
  if [[ $ENABLE_WORDPRESS == true ]]; then sources+=("${WP_DIR:-}"); fi
  if [[ $ENABLE_NEXTCLOUD == true ]]; then sources+=("${NC_DIR:-}" "${NC_DATA_DIR:-}"); fi
  if [[ $ENABLE_GITEA == true ]]; then sources+=("$GITEA_DATA_DIR" "$GITEA_ETC_DIR" "${GITEA_REPOS_DIR:-}" "${GITEA_CUSTOM_DIR:-}"); fi
  if [[ $ENABLE_MAIL == true ]]; then
    if [[ $ENABLE_MAIL_STORE == true ]]; then sources+=("$MAIL_DIR"); fi
    sources+=("$POSTFIX_DIR" "$DOVECOT_DIR")
    if [[ $ENABLE_MAIL_HOME_DIRS == true ]]; then sources+=("${MAIL_HOME_DIRS[@]}"); fi
  fi
  # Validate the WHOLE plan before mkdir/chmod touches any configured directory.
  for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
    resolved=$(realpath -m -- "$p")
    case $resolved in /|/etc|/usr|/usr/local|/var|/var/lib|/var/log|/var/backups|/home|/root|/tmp|/run|/opt|/srv)
      die "Use a dedicated working directory, not $resolved" ;;
    esac
    for src in "${sources[@]}"; do
      [[ -n $src ]] || continue
      other=$(realpath -m -- "$src")
      [[ $resolved != "$other" && $resolved != "$other/"* && $other != "$resolved/"* ]] || die "Source and working path overlap: $src / $p"
    done
  done
}

init_runtime() {
  need python3; need flock
  guard_workspaces
  # shellcheck source=lib/locking.sh
  source "$LIB_DIR/locking.sh"
  acquire_backup_lock 1
  for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
    [[ $p == /* && $p != / && $p != *$'\n'* && $p != *$'\t'* ]] || die "Unsafe working path: $p"
    [[ ! -L $p ]] || die "Symlink working directory refused: $p"
    mkdir -p -- "$p"
    [[ $(stat -c %u "$p") == 0 ]] || die "Working directory must belong to root: $p"
    chmod 700 -- "$p"
  done
  RUN_ID="${ARCHIVE_PREFIX}_$(date -u +%Y-%m-%d_%H-%M-%S)_$(python3 -c 'import secrets; print(secrets.token_hex(4))')"
  LOG_FILE="$LOG_DIR/${OPERATION}_${RUN_ID}.log"
  RUN_STARTED_EPOCH=$(date +%s)
  BACKUP_PHASE=preflight
  UPLOAD_STATUS=NOT_STARTED
  if [[ $ENABLE_UPLOAD != true ]]; then UPLOAD_STATUS=DISABLED; fi
  # Journald already captures service stdout. No dependency on systemd-cat in terminals.
  # Keep the log pipe alive while systemd sends TERM to the whole control group.
  # tee exits on EOF after the recovery traps close their writers.
  exec {REPORT_STDOUT}>&1 {REPORT_STDERR}>&2
  exec > >(trap '' INT TERM; exec tee -a "$LOG_FILE") 2>&1
  REPORT_LOGGER_PID=$!
  echo "== $OPERATION $RUN_ID =="
  local security_context
  if security_context=$(id -Z 2>/dev/null); then echo "-- SELinux process context: $security_context"; fi
  need rsync; need tar; need "$COMPRESSOR"
  STOPPED_SERVICES=(); GUARDED_SERVICES=(); GUARD_FILES=(); GUARD_STAMPS=(); SERVICE_GUARD_DIR=''; NC_CHANGED=false; NC_WAS_ON=false; MUTATION_STARTED=false
  trap 'exit 130' INT
  trap 'exit 143' TERM
  trap finish EXIT
}

# Same canonical paths must not contain one another when backup writes into them.
check_source_path() {
  local src p
  src=$(realpath -e -- "$1") || die "Missing source: $1"
  [[ -d $src && $src != / ]] || die "Source must be a directory other than /: $1"
  [[ $src != *$'\n'* && $src != *$'\t'* ]] || die "Unsupported control character in path"
  for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
    p=$(realpath -m -- "$p")
    [[ $p != "$src" && $p != "$src/"* && $src != "$p/"* ]] || die "Source and working path overlap: $src / $p"
  done
}

# Build a stable component mapping. Restore destinations come ONLY from local config.
build_plan() {
  LABELS=(); SOURCES=(); ROOTS=(); DB_LABELS=(); DB_NAMES=(); DB_CNFS=()
  if [[ $ENABLE_WORDPRESS == true ]]; then add_source wordpress "${WP_DIR:?Set WP_DIR}"; add_db wordpress WP; fi
  if [[ $ENABLE_NEXTCLOUD == true ]]; then
    add_source nextcloud "${NC_DIR:?Set NC_DIR}"
    : "${NC_DATA_DIR:=$NC_DIR/data}"
    if [[ $ENABLE_NEXTCLOUD_DATA == true ]]; then add_source nextcloud-data "$NC_DATA_DIR"; fi
    add_db nextcloud NC
  fi
  if [[ $ENABLE_GITEA == true ]]; then
    add_source gitea "$GITEA_DATA_DIR"; add_source gitea-etc "$GITEA_ETC_DIR"
    [[ -z ${GITEA_REPOS_DIR:-} ]] || add_source gitea-repos "$GITEA_REPOS_DIR"
    [[ -z ${GITEA_CUSTOM_DIR:-} ]] || add_source gitea-custom "$GITEA_CUSTOM_DIR"
    add_db gitea GITEA
  fi
  if [[ $ENABLE_MAIL == true ]]; then
    if [[ $ENABLE_MAIL_STORE == true ]]; then add_source mail "$MAIL_DIR"; fi
    add_source postfix "$POSTFIX_DIR"; add_source dovecot "$DOVECOT_DIR"
    if [[ $ENABLE_MAIL_HOME_DIRS == true ]]; then
      local i=0
      for p in "${MAIL_HOME_DIRS[@]}"; do add_source "mail-home-$i" "$p"; i=$((i+1)); done
    fi
  fi
  local i=0
  for p in "${EXTRA_PATHS[@]}"; do add_source "extra-$i" "$p"; i=$((i+1)); done
  (( ${#LABELS[@]} > 0 )) || die "No enabled sources"
  if [[ $ENABLE_NEXTCLOUD == true ]]; then
    local nc_path data_path
    nc_path=$(realpath -m -- "$NC_DIR"); data_path=$(realpath -m -- "$NC_DATA_DIR")
    [[ $nc_path != "$data_path" && $nc_path != "$data_path/"* ]] || die 'Nextcloud data must not contain the code directory'
    if [[ $ENABLE_WORDPRESS == true ]]; then
      [[ $(realpath -m -- "$WP_DIR") != "$nc_path" ]] || die 'WordPress and Nextcloud must not share the same directory'
    fi
  fi
}
add_source() { LABELS+=("$1"); SOURCES+=("$2"); ROOTS+=("files/$1"); }
add_db() {
  [[ $ENABLE_DB_DUMPS == true ]] || return 0
  local n="${2}_DB_NAME" c="${2}_DB_CNF"
  [[ -n ${!n:-} && -n ${!c:-} ]] || die "Set $n and $c for enabled application"
  [[ ${!n} =~ ^[a-zA-Z0-9_][a-zA-Z0-9_-]*$ ]] || die "Unsupported database name: ${!n}"
  DB_LABELS+=("db-$1"); DB_NAMES+=("${!n}"); DB_CNFS+=("${!c}")
}

check_db_credentials() {
  local f
  for f in "${DB_CNFS[@]}"; do
    secure_file "$f"
    (( (8#$(stat -c %a "$f") & 8#077) == 0 )) || die "Use chmod 600 for DB credentials: $f"
  done
}

check_source_access() (
    # This subshell owns only a fresh private probe directory. Keep source data
    # untouched and clean probe objects even if an attribute operation fails.
    local probe p
    probe=$(mktemp -d "$STAGING_ROOT/.access-probe-XXXXXXXX")
    trap 'rm -rf -- "$probe"' EXIT
    # Test through rsync itself: under SELinux it can enter a different domain
    # than this root shell. A shell test -r / realpath alone misses that denial.
    # --dirs lists one directory level only, without reading file contents,
    # recursing through live mailboxes, or creating a destination.
    for p in "${SOURCES[@]}"; do
      rsync --list-only --dirs -- "$p/" 8>&- 9>&- >/dev/null || die "rsync cannot list source: $p; check permissions and SELinux audit records before retrying"
      mkdir "$probe/root"
      # Exclude ALL children: copy only root-directory metadata into the private
      # probe. Exercise chmod/chown/times/ACLs/xattrs, including SELinux labels,
      # before maintenance. This is not a recursive/full content access test.
      rsync -aHAX --numeric-ids --exclude='/*' -- "$p/" "$probe/root/" 8>&- 9>&- || die "rsync cannot preserve directory metadata for: $p; inspect SELinux/filesystem permissions"
      rm -rf -- "$probe/root"
    done
    echo '-- Source directory access and staging metadata probe passed'
)

preflight() {
  for p in "${SOURCES[@]}"; do check_source_path "$p"; done
  if [[ $OPERATION == backup ]]; then check_source_access; fi
  check_db_credentials
  if (( ${#DB_LABELS[@]} )); then
    need "$DB_DUMP_BIN"; need "$DB_CLIENT_BIN"
    for i in "${!DB_LABELS[@]}"; do
      "$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --execute='SELECT 1' >/dev/null
      # Restore uses different privileges; this prerequisite belongs to dumps.
      # Metadata only: never briefly lock the live database during --check.
      if [[ $OPERATION == backup ]]; then
        helper db-privileges "$DB_CLIENT_BIN" "${DB_CNFS[i]}" "${DB_NAMES[i]}"
      fi
    done
  fi
  if [[ $ENABLE_NEXTCLOUD == true ]]; then need sudo; need php; [[ -f $NC_DIR/occ ]] || die "Nextcloud occ missing"; fi
  need systemctl
  if [[ $ENABLE_UPLOAD == true ]]; then check_remote; fi
  check_space "$STAGING_ROOT"; check_space "$ARCHIVE_DIR"
}
check_space() {
  local avail
  avail=$(df -PB1 -- "$1" | awk 'END{print $4}')
  (( avail >= MIN_FREE_GB * 1024 * 1024 * 1024 )) || die "Free-space reserve (${MIN_FREE_GB} GiB) unavailable on $1"
}

check_remote() {
  need "$RCLONE_BIN"
  need timeout
  [[ $RCLONE_REMOTE_BASE =~ ^[a-zA-Z0-9_-]+:[^:]+$ ]] || die "Use a named crypt remote with a dedicated non-empty path"
  [[ $RCLONE_REMOTE_BASE != *$'\n'* ]] || die "Invalid remote path"
  secure_file "$RCLONE_CONFIG"
  (( (8#$(stat -c %a "$RCLONE_CONFIG") & 8#077) == 0 )) || die "Use chmod 600 for rclone config"
  # Environment-defined backend types/settings could invalidate a config-only audit.
  local key name=${RCLONE_REMOTE_BASE%%:*}
  for key in ${!RCLONE_CONFIG_@}; do
    [[ $key == RCLONE_CONFIG_PASS ]] || die "Remote overrides through $key are unsupported; use the config file"
  done
  # rclone reads global flags from exported RCLONE_* variables too. Filters
  # can hide missing archives from both copy and check. Refuse such ambient flags
  # before the first remote operation, rather than silently weakening verification.
  # Allow only our explicit settings (passed as CLI flags) and config decryption.
  while IFS= read -r key; do
    [[ $key == RCLONE_* ]] || continue
    case $key in
      RCLONE_BIN|RCLONE_CONFIG|RCLONE_CONFIG_PASS|RCLONE_REMOTE_BASE|RCLONE_RETRIES|\
      RCLONE_LOW_LEVEL_RETRIES|RCLONE_RETRIES_SLEEP|RCLONE_TIMEOUT|RCLONE_CONTIMEOUT|\
      RCLONE_CHECK_ATTEMPTS|RCLONE_CHECK_RETRY_DELAY|RCLONE_CHECK_TIMEOUT|RCLONE_CHECK_DEADLINE|RCLONE_CHECK_LOW_LEVEL_RETRIES|\
      RCLONE_STATS_INTERVAL|RCLONE_TRANSFERS|RCLONE_CHECKERS|RCLONE_ONEDRIVE_CHUNK_SIZE|RCLONE_BWLIMIT) ;;
      *) die "Unsupported rclone environment override: $key; use documented app-backup settings and the remote config" ;;
    esac
  done < <(compgen -e)
  # Pipe directly into the parser: compatibility with older rclone without
  # config redacted. Secret values never go to the log or a temporary file.
  "$RCLONE_BIN" --ask-password=false --config "$RCLONE_CONFIG" config show "$name" | helper crypt "$name"
  RCLONE_ARGS=(--ask-password=false --config "$RCLONE_CONFIG" --no-check-certificate=false --crypt-filename-encryption standard
    --crypt-directory-name-encryption=true --crypt-no-data-encryption=false
    --retries "$RCLONE_RETRIES" --low-level-retries "$RCLONE_LOW_LEVEL_RETRIES"
    --retries-sleep "$RCLONE_RETRIES_SLEEP" --timeout "$RCLONE_TIMEOUT" --contimeout "$RCLONE_CONTIMEOUT"
    --stats "$RCLONE_STATS_INTERVAL" --stats-one-line --stats-log-level NOTICE
    --transfers "$RCLONE_TRANSFERS" --checkers "$RCLONE_CHECKERS" --onedrive-chunk-size "$RCLONE_ONEDRIVE_CHUNK_SIZE"
    --bwlimit "$RCLONE_BWLIMIT")
  rc lsd "${name}:" --max-depth 1 >/dev/null
}
# Statistics go to stderr, leaving stdout intact for manifests/markers captured
# by callers. NOTICE makes periodic progress visible without verbose HTTP logs.
rc() { "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@"; }

remote_metadata_check() {
  local attempt code=1
  # rclone check uses cmd.Run(false, ...): --retries does NOT repeat the whole
  # check. Retry the complete metadata comparison; never download archive data.
  # Content integrity is checked by the monthly restore test.
  for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
    echo "-- Remote metadata check attempt $attempt/$RCLONE_CHECK_ATTEMPTS (deadline ${RCLONE_CHECK_DEADLINE}s, idle ${RCLONE_CHECK_TIMEOUT}s)"
    # GNU timeout bounds backend pacing/retries as well as stalled I/O. It owns
    # a process group so TERM/KILL cannot leave an old check running on retry.
    # Combined output names missing files and size mismatches.
    if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
        "$RCLONE_BIN" "${RCLONE_ARGS[@]}" check "$@" --size-only \
        --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" \
        --stats-one-line=false --stats-one-line-date=false --combined -; then
      echo "-- Remote metadata check attempt $attempt passed (all selected files)"
      return 0
    else
      code=$?
    fi
    echo "WARNING: Remote metadata check attempt $attempt/$RCLONE_CHECK_ATTEMPTS failed (exit $code; 124 means deadline exceeded)"
    # Preserve cancellation semantics, including a forced kill after the grace
    # period. Do not restart a command an operator/service manager has stopped.
    if (( code >= 128 )); then return "$code"; fi
    if (( attempt < RCLONE_CHECK_ATTEMPTS )); then
      sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?
    fi
  done
  echo 'ERROR: Remote metadata check exhausted; remote file presence/sizes remain unverified'
  return "$code"
}

# Before reclaiming a local copy, bind the remote sizes to the exact local
# manifest and completion marker. No archive download or temporary disk space.
remote_identity_check() {
  local dir=$1 remote=$2 file attempt code size
  local -a statuses
  need cmp
  for file in manifest.json COMPLETE; do
    size=$(stat -c %s -- "$dir/$file") || return 1
    code=1
    for ((attempt=1; attempt<=RCLONE_CHECK_ATTEMPTS; attempt++)); do
      if timeout --kill-after=10s "${RCLONE_CHECK_DEADLINE}s" \
          "$RCLONE_BIN" "${RCLONE_ARGS[@]}" cat "$remote/$file" --head "$((size + 1))" \
          --timeout "${RCLONE_CHECK_TIMEOUT}s" --low-level-retries "$RCLONE_CHECK_LOW_LEVEL_RETRIES" |
          cmp -s -- "$dir/$file" -; then
        code=0
        break
      else
        statuses=("${PIPESTATUS[@]}")
        code=${statuses[0]}
        if (( code == 0 )); then code=${statuses[1]}; fi
      fi
      (( code < 128 )) || return "$code"
      echo "WARNING: Remote identity check failed: $file (attempt $attempt/$RCLONE_CHECK_ATTEMPTS)"
      if (( attempt < RCLONE_CHECK_ATTEMPTS )); then sleep "$RCLONE_CHECK_RETRY_DELAY" || return $?; fi
    done
    (( code == 0 )) || return "$code"
  done
}

# Exclusions are rooted so a nested directory named data is not accidentally lost.
exclusions() {
  EXCLUDES=()
  local rel x
  if [[ $1 == wordpress ]]; then
    for x in "${NC_DIR:-}" "${NC_DATA_DIR:-}"; do
      [[ -n $x ]] || continue
      rel=$(realpath -m --relative-to="${WP_DIR}" -- "$x")
      if [[ $rel != .. && $rel != ../* && $rel != . ]]; then EXCLUDES+=("--exclude=/$rel/***"); fi
    done
    for x in "${WP_EXCLUDES[@]}"; do EXCLUDES+=("--exclude=$x"); done
  elif [[ $1 == nextcloud ]]; then
    rel=$(realpath -m --relative-to="$NC_DIR" -- "$NC_DATA_DIR")
    if [[ $rel != .. && $rel != ../* && $rel != . ]]; then EXCLUDES+=("--exclude=/$rel/***"); fi
  elif [[ $1 == mail-home-* ]]; then
    EXCLUDES+=(--one-file-system)
    for x in "${MAIL_HOME_EXCLUDES[@]}"; do EXCLUDES+=("--exclude=${x#--exclude=}"); done
  fi
}

# A stop alone does not exclude timer/monitor/dependency activation. Attach a
# run-owned runtime condition before stopping each writer. Unlike a runtime mask,
# a drop-in also works for administrator units whose main file lives in /etc.
# This runtime root is overridable only by the trusted config (isolated tests).
install_service_guard() {
  local unit=$1 id load dir file paths existing
  for id in "${GUARDED_SERVICES[@]}"; do [[ $id != "$unit" ]] || return 0; done
  load=$(systemctl show "$unit" --property=LoadState --value)
  [[ $load == loaded || $load == masked ]] || die "Cannot guard unloaded unit: $unit ($load)"
  if [[ $load == masked ]]; then
    # Preserve pre-existing masks verbatim; never claim ownership of them.
    GUARDED_SERVICES+=("$unit"); GUARD_FILES+=(''); return 0
  fi
  id=$(systemctl show "$unit" --property=Id --value)
  [[ $id =~ ^[a-zA-Z0-9_@:.\\-]+\.(service|timer|socket|path)$ ]] || die "Unsupported writer unit name: $id"
  : "${SERVICE_GUARD_RUNTIME_ROOT:=/run}"
  [[ $SERVICE_GUARD_RUNTIME_ROOT =~ ^/[a-zA-Z0-9_./-]+$ ]] || die 'Unsafe service guard runtime root'
  if [[ -z $SERVICE_GUARD_DIR ]]; then
    SERVICE_GUARD_DIR=$(mktemp -d "$SERVICE_GUARD_RUNTIME_ROOT/app-backup-guard-XXXXXXXX")
  fi
  dir="$SERVICE_GUARD_RUNTIME_ROOT/systemd/system/$id.d"
  [[ ! -L $dir ]] || die "Symlink writer drop-in directory refused: $dir"
  mkdir -p -- "$dir"
  [[ $(stat -c %u "$dir") == 0 ]] || die "Writer drop-in directory must belong to root: $dir"
  (( (8#$(stat -c %a "$dir") & 8#022) == 0 )) || die "Unsafe writer drop-in directory: $dir"
  file="$dir/zzzz-app-backup-$RUN_ID.conf"
  for existing in "${GUARD_FILES[@]}"; do [[ $existing != "$file" ]] || return 0; done
  [[ ! -e $file && ! -L $file ]] || die "Guard file already exists: $file"
  GUARDED_SERVICES+=("$unit"); GUARD_FILES+=("$file")
  # Record our exact paths BEFORE changing systemd state, including on failures.
  save_recovery
  (set -o noclobber; printf '[Unit]\nConditionPathExists=!%s\n' "$SERVICE_GUARD_DIR" > "$file")
  systemctl daemon-reload
  paths=$(systemctl show "$unit" --property=DropInPaths --value)
  # Requiring our drop-in last avoids a later Condition*= resetting the guard.
  [[ ${paths##* } == "$file" ]] || die "Start guard not loaded last for $unit; review its drop-ins"
  echo "-- Start guard installed: $unit"
}

assert_services_quiescent() {
  local i unit state stamp paths
  for i in "${!GUARDED_SERVICES[@]}"; do
    unit=${GUARDED_SERVICES[i]}
    state=$(systemctl show "$unit" --property=ActiveState --value)
    [[ $state == inactive || $state == failed ]] || die "Consistency lost: writer $unit became $state during the copy window"
    if [[ -n ${GUARD_FILES[i]} ]]; then
      [[ -d $SERVICE_GUARD_DIR && -f ${GUARD_FILES[i]} ]] || die "Consistency lost: start guard removed for $unit"
      [[ $(cat -- "${GUARD_FILES[i]}") == "$(printf '[Unit]\nConditionPathExists=!%s' "$SERVICE_GUARD_DIR")" ]] || die "Consistency lost: start guard changed for $unit"
      paths=$(systemctl show "$unit" --property=DropInPaths --value)
      [[ ${paths##* } == "${GUARD_FILES[i]}" ]] || die "Consistency lost: start guard overridden for $unit"
    else
      [[ $(systemctl show "$unit" --property=LoadState --value) == masked ]] || die "Consistency lost: existing mask removed for $unit"
    fi
    stamp=$(systemctl show "$unit" --property=ActiveEnterTimestampMonotonic --property=InactiveExitTimestampMonotonic)
    if [[ -n ${GUARD_STAMPS[i]:-} && $stamp != "${GUARD_STAMPS[i]}" ]]; then
      die "Consistency lost: writer $unit was restarted during the copy window"
    fi
    [[ -n $stamp ]] || die "Missing writer state timestamps: $unit"
    GUARD_STAMPS[i]=$stamp
  done
}

release_service_guards() {
  local file
  # Remove only this run's drop-ins. Keep the condition directory until reload
  # succeeds, so stale manager state continues to refuse starts on failure.
  for file in "${GUARD_FILES[@]}"; do
    [[ -z $file ]] || rm -f -- "$file" || return 1
  done
  if (( ${#GUARDED_SERVICES[@]} )); then systemctl daemon-reload || return 1; fi
  if [[ -n $SERVICE_GUARD_DIR ]]; then rmdir -- "$SERVICE_GUARD_DIR" || return 1; fi
  SERVICE_GUARD_DIR=''; GUARDED_SERVICES=(); GUARD_FILES=(); GUARD_STAMPS=()
}

stop_service() {
  local state
  install_service_guard "$1"
  state=$(systemctl show "$1" --property=ActiveState --value)
  case $state in
    active|activating|reloading)
      # Record intent before stop: a failed/partial stop still needs recovery.
      STOPPED_SERVICES+=("$1")
      save_recovery
      echo "-- Stopping service: $1 (waiting for systemd)"
      # Conditions block activation, but a concurrent reload/start can still
      # replace a pending stop job before conditions are evaluated. Protect the
      # job until it finishes; this is not a persistent mask and does not alter
      # stop timeouts or the normal recovery start after releasing our guards.
      if ! systemctl stop "$1" --job-mode=replace-irreversibly; then
        die "Could not stop writer $1; inspect its systemd journal (stop failure or explicit job cancellation)"
      fi
      echo "-- Service stopped: $1"
      ;;
    inactive|failed) ;;
    *) die "Unknown/transitioning service state for $1: $state" ;;
  esac
}
occ() { sudo -u "$NC_OCC_USER" php "$NC_DIR/occ" "$@"; }
quiesce() {
  if [[ $ENABLE_NEXTCLOUD == true ]]; then
    if [[ $ENABLE_NEXTCLOUD_MAINTENANCE == true ]]; then
      local status
      status=$(occ status --output=json)
      NC_WAS_ON=$(printf '%s' "$status" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert isinstance(d["maintenance"], bool); print(str(d["maintenance"]).lower())')
      if [[ $NC_WAS_ON == false ]]; then NC_CHANGED=true; save_recovery; occ maintenance:mode --on; fi
    else echo "WARNING: Nextcloud maintenance disabled; consistency depends on external writer shutdown"; fi
  fi
  # Cron daemons are included only if explicitly configured by the operator.
  for p in "${QUIESCE_SERVICES[@]}"; do stop_service "$p"; done
  if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then
    if [[ $ENABLE_HTTPD_STOP == true ]]; then stop_service "$HTTPD_SERVICE_NAME"; fi
    if [[ $ENABLE_PHPFPM_STOP == true ]]; then stop_service "$PHPFPM_SERVICE_NAME"; fi
  fi
  if [[ $ENABLE_GITEA == true ]]; then
    if [[ $ENABLE_GITEA_SERVICE_STOP == true ]]; then stop_service "$GITEA_SERVICE_NAME"; else echo "WARNING: Gitea is not stopped"; fi
  fi
  if [[ $ENABLE_MAIL == true ]]; then
    if [[ $MAIL_BACKUP_MODE == stop ]]; then stop_service postfix; stop_service dovecot
    else echo "WARNING: live mail copy is not a consistent snapshot"; fi
  fi
  if [[ $ENABLE_NEXTCLOUD == true ]]; then
    # Cron daemons may leave an already launched PHP child running after stop.
    # Wait for direct cron.php / occ processes to finish; never kill them blindly.
    echo '-- Waiting for remaining Nextcloud writers'
    helper wait-nextcloud "$NC_DIR" "${WRITER_DRAIN_TIMEOUT:=300}"
    echo '-- Nextcloud writers finished'
  fi
  assert_services_quiescent
  echo '-- All configured writers stopped and protected against activation'
}
# Survives SIGKILL/power loss, which Bash traps cannot handle. This is an
# instruction record for the operator, not an automatically executed recovery.
save_recovery() {
  python3 - "$WORKDIR/recovery-$RUN_ID.json" "$OPERATION" "$NC_CHANGED" "$NC_WAS_ON" "${STAGING_DIR:-}" "$SERVICE_GUARD_DIR" "${#GUARD_FILES[@]}" "${GUARD_FILES[@]}" "${STOPPED_SERVICES[@]}" <<'PYREC'
import json,os,sys
path,operation,changed,was_on,staging,guard_dir,count,*items=sys.argv[1:]
guard_files=items[:int(count)]
services=items[int(count):]
with open(path+'.tmp','w') as stream:
    json.dump(dict(operation=operation, staging=staging, nextcloud_changed=changed=='true', nextcloud_was_on=was_on=='true', stopped_services=services, service_guard_directory=guard_dir, service_guard_files=[p for p in guard_files if p]),stream,indent=2)
    stream.flush()
    os.fsync(stream.fileno())
os.replace(path+'.tmp',path)
PYREC
}
resume_services() {
  local failed=0 i
  # Remove all our guards before starting dependencies in reverse order. Never
  # start anything if guard cleanup/reload failed: retain manual recovery state.
  if ! release_service_guards; then
    echo 'ERROR: Could not release writer start guards; manual recovery required'
    save_recovery
    return 1
  fi
  if [[ $NC_CHANGED == true ]]; then
    if occ maintenance:mode --off; then NC_CHANGED=false; else failed=1; fi
  fi
  for ((i=${#STOPPED_SERVICES[@]}-1; i>=0; i--)); do
    [[ -n ${STOPPED_SERVICES[i]} ]] || continue
    echo "-- Starting service: ${STOPPED_SERVICES[i]} (waiting for systemd)"
    if systemctl start "${STOPPED_SERVICES[i]}" &&
       [[ $(systemctl show "${STOPPED_SERVICES[i]}" --property=ActiveState --value) == active ]]; then
      echo "-- Service started: ${STOPPED_SERVICES[i]}"
      STOPPED_SERVICES[i]=''
    else
      echo "ERROR: Service did not become active: ${STOPPED_SERVICES[i]}"
      failed=1
    fi
  done
  if (( failed == 0 )); then
    rm -f -- "$WORKDIR/recovery-$RUN_ID.json"
  else save_recovery; fi
  return "$failed"
}
finish() {
  local code=$?
  local recovery_ok=true
  trap - EXIT INT TERM
  set +e
  if [[ $OPERATION == restore && $MUTATION_STARTED == true && $code != 0 ]]; then
    if [[ $ENABLE_NEXTCLOUD == true ]]; then occ maintenance:mode --on || echo 'ERROR: Could not enforce Nextcloud maintenance'; fi
    save_recovery
    echo "ERROR: Partial restore. Service recovery requires manual review: ${STOPPED_SERVICES[*]}"
    echo "ERROR: Keep Nextcloud in maintenance; review $LOG_FILE before restarting."
  else
    if ! resume_services; then code=1; recovery_ok=false; fi
  fi
  local report_sizes="$LOG_DIR/report_$RUN_ID.sizes.json"
  if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
    # Collect after service recovery but before successful staging cleanup.
    python3 "$LIB_DIR/report.py" snapshot "$report_sizes" "${STAGING_DIR:-}" "${RUN_DIR:-${PARTIAL_DIR:-}}" ||
      echo 'WARNING: Report sizes unavailable'
  fi
  if [[ $OPERATION == backup && $recovery_ok == true && -n ${STAGING_DIR:-} && ! -e $WORKDIR/recovery-$RUN_ID.json ]]; then
    # A failed backup scratch copy is disposable once writer recovery succeeded.
    # Never delete a locally complete set awaiting upload or a completion marker.
    if [[ -n ${PARTIAL_DIR:-} && -d $PARTIAL_DIR && ! -e $PARTIAL_DIR/COMPLETE ]]; then
      python3 "$LIB_DIR/space.py" remove "$PARTIAL_DIR" || code=1
    fi
    if [[ ! -d ${PARTIAL_DIR:-/nonexistent} ]]; then
      python3 "$LIB_DIR/space.py" remove "$STAGING_DIR" && STAGING_DIR='' || code=1
    fi
  elif [[ ( $code == 0 || ( $OPERATION == restore && ${MODE:-} == verify ) ) && -n ${STAGING_DIR:-} ]]; then
    rm -rf -- "$STAGING_DIR" && STAGING_DIR='' || code=1
  fi
  if [[ $code != 0 ]]; then
    echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none} (temporary staging only)"
    if [[ $OPERATION == backup && -n ${RUN_DIR:-} && -f $RUN_DIR/COMPLETE ]]; then
      echo "Retained local backup: $RUN_DIR"
      if [[ $ENABLE_UPLOAD == true && ! -f $RUN_DIR/UPLOADED ]]; then
        echo 'Remote publication not confirmed; local backup retained for recovery.'
      fi
    fi
  fi
  # Close the logger's writers and wait for EOF so the mail tail includes the
  # final recovery/status messages. Reporting itself uses the original stdout.
  exec 1>&"$REPORT_STDOUT" 2>&"$REPORT_STDERR"
  exec {REPORT_STDOUT}>&- {REPORT_STDERR}>&-
  wait "$REPORT_LOGGER_PID" || code=1
  # Persist status separately from --check; failed status writes also fail the job.
  if [[ ${CHECK:-false} != true ]]; then
    helper status "$WORKDIR" "$OPERATION" "$RUN_ID" "$code" || code=1
  fi
  if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
    python3 "$LIB_DIR/report.py" deliver --sizes "$report_sizes" --log "$LOG_FILE" \
      --output "$LOG_DIR/report_$RUN_ID.eml" --to "$MAIL_TO" --sender "$MAIL_FROM" \
      --hostname="$MAIL_HOSTNAME" --subject-prefix "$MAIL_SUBJECT_PREFIX" --run "$RUN_ID" --exit-code "$code" \
      --started "$RUN_STARTED_EPOCH" --phase "$BACKUP_PHASE" --upload-status "$UPLOAD_STATUS" \
      --config "$CONFIG_FILE" --workdir "$WORKDIR" --archive-dir "$ARCHIVE_DIR" \
      --staging-root "$STAGING_ROOT" --restore-root "$RESTORE_ROOT" --log-dir "$LOG_DIR" \
      --compression "$COMPRESSOR" --remote "$RCLONE_REMOTE_BASE" --sendmail "$MAIL_SENDMAIL_BIN" \
      --tail-lines "$MAIL_INCLUDE_LOG_TAIL_LINES" --timeout "$MAIL_SEND_TIMEOUT" 2>&1 | tee -a "$LOG_FILE"
    rm -f -- "$report_sizes"
  fi
  exit "$code"
}