app-backup/update.sh

#!/usr/bin/env bash
# Upgrade coordinator: prepare locally, then run the long operation under systemd.
# No Git checkout, credential template replacement, or application restore needed.
set -Eeuo pipefail
umask 077
UPDATE_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
LIB_DIR="$UPDATE_ROOT/lib"
# shellcheck source=lib/common.sh
source "$LIB_DIR/common.sh"
# shellcheck source=lib/locking.sh
source "$LIB_DIR/locking.sh"
CONFIG_FILE=/etc/app-backup/app-backup.conf
UP_FOREGROUND=false
UP_INSTALL_DEPS=false
UP_GITEA=''
UP_RCLONE=''
UP_CRON=''
UP_MAIL_DIR=''
UP_MAIL_HOMES=false
UP_STATE=''
UP_WORKER=false
while (( $# )); do
  case $1 in
    --config) CONFIG_FILE=${2:?Missing configuration}; shift 2 ;;
    --foreground) UP_FOREGROUND=true; shift ;;
    --install-deps) UP_INSTALL_DEPS=true; shift ;;
    --enable-gitea) UP_GITEA=true; shift ;;
    --no-gitea) UP_GITEA=false; shift ;;
    --rclone-config) UP_RCLONE=${2:?Missing rclone configuration}; shift 2 ;;
    --cron-services) UP_CRON=${2:?Missing comma-separated systemd units}; shift 2 ;;
    --mail-dir) UP_MAIL_DIR=${2:?Missing mailbox directory}; shift 2 ;;
    --mail-homes-only) UP_MAIL_HOMES=true; shift ;;
    --worker) UP_WORKER=true; UP_STATE=${2:?Missing state}; shift 2 ;;
    --help) echo 'Usage: sudo bash update.sh [--foreground] [--config FILE] [--rclone-config FILE] [--enable-gitea | --no-gitea] [--cron-services UNIT,...] [--mail-dir DIR | --mail-homes-only] [--install-deps]'; exit 0 ;;
    *) die "Unknown argument: $1" ;;
  esac
done
[[ -z $UP_MAIL_DIR || $UP_MAIL_HOMES == false ]] || die 'Choose --mail-dir OR --mail-homes-only'
[[ $EUID == 0 ]] || die 'Run with sudo bash'
need systemctl; need flock
# This lock excludes two coordinators, independently from ordinary backup jobs.
exec 8>/run/app-backup-upgrade.lock
# Workers install their persistent error reporting before waiting on this lock.
if [[ $UP_WORKER == false ]]; then flock -n 8 || die 'Another updater is preparing/running'; fi

prompt() {
  local answer
  [[ -t 0 ]] || die "$1 (noninteractive: supply the matching command-line option/configuration)"
  read -r -p "$1 " answer
  [[ $answer == j || $answer == J || $answer == ja || $answer == y || $answer == yes ]]
}
append_setting() { printf '%s=%q\n' "$1" "$2" >> "$UP_STATE/candidate.conf"; }
discover() { python3 "$LIB_DIR/update-discover.py" "$@"; }

# Do not infer mailbox coverage from an existing directory. In particular,
# creating an empty /var/vmail would make preflight pass without saving mail.
prepare_mail() {
  local choice home
  if [[ $ENABLE_MAIL != true ]]; then
    [[ -z $UP_MAIL_DIR && $UP_MAIL_HOMES == false ]] || die 'Mail options require ENABLE_MAIL=true'
    return 0
  fi
  if [[ -n $UP_MAIL_DIR ]]; then
    MAIL_DIR=$UP_MAIL_DIR
    ENABLE_MAIL_STORE=true
  elif [[ $UP_MAIL_HOMES == true ]]; then
    ENABLE_MAIL_STORE=false
  elif [[ $ENABLE_MAIL_STORE == true && ! -d $MAIL_DIR ]]; then
    echo "Konfiguriertes zentrales Mailverzeichnis fehlt: $MAIL_DIR"
    printf 'Konfigurierte Mail-Homes: '; printf '%q ' "${MAIL_HOME_DIRS[@]}"; echo
    echo 'Absoluten Pfad zur tatsaechlichen zentralen Mailablage angeben.'
    echo 'Nur wenn ALLE Mailboxen in den konfigurierten Homes liegen: homes eingeben.'
    echo 'Bei unbekannter Ablage abbrechen; kein leeres Ersatzverzeichnis anlegen.'
    [[ -t 0 ]] || die 'Missing mail source: supply --mail-dir DIR or explicitly confirm coverage with --mail-homes-only'
    read -r -p 'Mailverzeichnis / homes / Enter zum Abbrechen: ' choice
    [[ -n $choice ]] || die 'Mail storage unresolved; installed package/config/timer unchanged'
    if [[ $choice == homes ]]; then
      printf 'Home-Excludes: '; printf '%q ' "${MAIL_HOME_EXCLUDES[@]}"; echo
      prompt 'Alle Mailboxen liegen in diesen Homes, ohne ausgeschlossene Pfade oder weitere Mounts? [j/N]' || die 'Mail coverage not confirmed'
      ENABLE_MAIL_STORE=false
    else MAIL_DIR=$choice; fi
  fi
  if [[ $ENABLE_MAIL_STORE == true ]]; then
    [[ $MAIL_DIR == /* ]] || die 'MAIL_DIR must be absolute'
    check_source_path "$MAIL_DIR"
    append_setting MAIL_DIR "$MAIL_DIR"
    echo "Zentrales Mailverzeichnis: $MAIL_DIR"
  else
    [[ $ENABLE_MAIL_HOME_DIRS == true && ${#MAIL_HOME_DIRS[@]} -gt 0 ]] || die 'Homes-only mail requires enabled, nonempty MAIL_HOME_DIRS'
    for home in "${MAIL_HOME_DIRS[@]}"; do
      [[ $home == /* ]] || die 'MAIL_HOME_DIRS must be absolute'
      check_source_path "$home"
    done
    printf 'Mail ausschliesslich in Homes (Excludes und Mountgrenzen gelten): '
    printf '%q ' "${MAIL_HOME_DIRS[@]}"; echo
  fi
  append_setting ENABLE_MAIL_STORE "$ENABLE_MAIL_STORE"
}

if [[ $UP_WORKER == false ]]; then
  systemctl is-active --quiet app-backup-upgrade.service && die 'Upgrade service is already running'
  # Only install missing, small prerequisites when requested. Never upgrade PHP,
  # database server or rclone to a guessed version as a side effect of this update.
  missing=()
  for pair in python3:python3 rsync:rsync zstd:zstd gzip:gzip tar:tar sudo:sudo; do
    tool=${pair%%:*}; package=${pair#*:}
    command -v "$tool" >/dev/null || missing+=("$package")
  done
  if (( ${#missing[@]} )); then
    if [[ $UP_INSTALL_DEPS == true ]] || prompt "Fehlende Basispakete installieren: ${missing[*]}? [j/N]"; then
      need dnf
      dnf install -y "${missing[@]}"
    else die "Missing prerequisites: ${missing[*]}"; fi
  fi
  [[ $CONFIG_FILE == /etc/app-backup/app-backup.conf ]] || die 'The service uses /etc/app-backup/app-backup.conf; migrate that canonical configuration'
  secure_file "$CONFIG_FILE"
  UP_STATE=$(mktemp -d /var/backups/app-backup-upgrade-XXXXXXXX)
  chmod 700 "$UP_STATE"
  cp -a -- "$CONFIG_FILE" "$UP_STATE/original.conf"
  printf '%s\n' "$CONFIG_FILE" > "$UP_STATE/config-path"
  sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1 > "$UP_STATE/original.sha256"
  cp -a /etc/app-backup "$UP_STATE/etc-app-backup"
  echo "Update-Sicherung und Protokoll: $UP_STATE"
  # Read only trusted local Bash configuration; remote metadata is never sourced.
  # shellcheck disable=SC1090
  source "$CONFIG_FILE"
  UP_HAD_GITEA=${ENABLE_GITEA+x}
  UP_ORIGINAL_RCLONE=${RCLONE_CONFIG:-}
  load_config
  acquire_backup_lock
  # Preserve original text and append effective safety settings. Repeated upgrades
  # may append another small block; the last explicit values remain authoritative.
  cp -- "$CONFIG_FILE" "$UP_STATE/candidate.conf"
  chmod 600 "$UP_STATE/candidate.conf"
  printf '\n# Automated package migration (original preserved in update snapshot)\n' >> "$UP_STATE/candidate.conf"
  append_setting ENABLE_REMOTE_RETENTION false
  # Retention of existing local sets is also skipped for the acceptance backup.
  append_setting ENABLE_UPLOAD true
  append_setting ENABLE_NEXTCLOUD_MAINTENANCE true
  append_setting MAIL_BACKUP_MODE stop
  prepare_mail
  if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then
    for pair in "ENABLE_HTTPD_STOP:$HTTPD_SERVICE_NAME" "ENABLE_PHPFPM_STOP:$PHPFPM_SERVICE_NAME"; do
      name=${pair%%:*}; unit=${pair#*:}
      state=$(systemctl show "$unit" --property=LoadState --value)
      if [[ $state == loaded ]]; then append_setting "$name" true
      elif [[ $state == not-found ]]; then append_setting "$name" false
      else die "Cannot inspect web service: $unit ($state)"; fi
    done
  fi
  if [[ -n $UP_GITEA ]]; then
    ENABLE_GITEA=$UP_GITEA
  elif [[ -z $UP_HAD_GITEA && $(systemctl show "$GITEA_SERVICE_NAME" --property=LoadState --value) == loaded ]]; then
    if prompt 'Gitea wurde gefunden, bisher aber nicht explizit konfiguriert. In die Sicherung aufnehmen? [j/N]'; then ENABLE_GITEA=true
    else die 'Choose --no-gitea explicitly if Gitea should remain excluded'; fi
  fi
  if [[ $ENABLE_GITEA == true && -z $UP_HAD_GITEA ]]; then
    [[ -f $GITEA_ETC_DIR/app.ini ]] || die 'Gitea app.ini missing; configure Gitea paths first'
    discover gitea "$GITEA_ETC_DIR/app.ini" > "$UP_STATE/gitea-paths.conf"
    # Output contains ONLY allowlisted shell-quoted path/DB-name assignments.
    # shellcheck disable=SC1090,SC1091
    source "$UP_STATE/gitea-paths.conf"
    [[ -n ${GITEA_REPOS_DIR:-} && -n ${GITEA_DB_NAME:-} ]] || die 'Gitea repository path/database not explicit; set GITEA_REPOS_DIR and GITEA_DB_NAME in the existing config'
    : "${GITEA_DB_CNF:=/etc/app-backup/db-gitea.cnf}"
    for name in GITEA_DATA_DIR GITEA_ETC_DIR GITEA_REPOS_DIR GITEA_DB_NAME GITEA_DB_CNF; do append_setting "$name" "${!name}"; done
  fi
  append_setting ENABLE_GITEA "$ENABLE_GITEA"
  append_setting ENABLE_GITEA_SERVICE_STOP true
  [[ $ENABLE_GITEA == true ]] || echo 'Hinweis: Gitea bleibt entsprechend der Konfiguration deaktiviert.'

  # Keep keys/tokens. If a user-owned config was used before, copy it into a
  # dedicated root-owned file; never chown a user's actively used configuration.
  if [[ -n $UP_RCLONE ]]; then RCLONE_CONFIG=$UP_RCLONE
  elif [[ -n $UP_ORIGINAL_RCLONE ]]; then RCLONE_CONFIG=$UP_ORIGINAL_RCLONE
  else
    detected=$(discover rclone-config)
    if [[ -n $detected ]]; then RCLONE_CONFIG=$detected
    else
      RCLONE_CONFIG=$("$RCLONE_BIN" config file | tail -n 1)
    fi
  fi
  if [[ ! -f $RCLONE_CONFIG || -L $RCLONE_CONFIG ]]; then
    [[ -t 0 ]] || die 'rclone config not found; supply --rclone-config FILE'
    read -r -p 'Pfad zur bisher verwendeten rclone.conf: ' RCLONE_CONFIG
  fi
  [[ -f $RCLONE_CONFIG && ! -L $RCLONE_CONFIG ]] || die 'rclone config is not a regular file'
  cp -- "$RCLONE_CONFIG" "$UP_STATE/rclone.conf"
  chmod 600 "$UP_STATE/rclone.conf"
  # Use the original root-owned path where possible, preserving OAuth refreshes.
  if [[ $(stat -c %u "$RCLONE_CONFIG") != 0 ]]; then
    RCLONE_CONFIG=/etc/app-backup/rclone-migrated.conf
    [[ ! -e $RCLONE_CONFIG ]] || die 'rclone-migrated.conf already exists; specify it explicitly'
  fi
  append_setting RCLONE_CONFIG "$RCLONE_CONFIG"
  printf '%s\n' "$RCLONE_CONFIG" > "$UP_STATE/rclone-path"

  if [[ -n $UP_CRON ]]; then IFS=, read -r -a additional <<< "$UP_CRON"; QUIESCE_SERVICES+=("${additional[@]}"); fi
  if [[ $ENABLE_NEXTCLOUD == true ]]; then
    discover cron-units "$NC_DIR" > "$UP_STATE/cron-units"
    mapfile -t detected_units < "$UP_STATE/cron-units"
    QUIESCE_SERVICES+=("${detected_units[@]}")
    # Arbitrary cron wrappers cannot be safely translated into systemd timers.
    # Offer a precise policy choice rather than silently omit an active writer.
    cron_result=0
    discover check-crontabs "$NC_DIR" "$NC_OCC_USER" || cron_result=$?
    [[ $cron_result == 0 || $cron_result == 2 ]] || die 'Cron inventory failed; no safe writer plan available'
    if [[ $cron_result == 2 ]]; then
      cron_unit=crond.service
      if [[ $(systemctl show "$cron_unit" --property=LoadState --value) != loaded ]]; then cron_unit=cron.service; fi
      [[ $(systemctl show "$cron_unit" --property=LoadState --value) == loaded ]] || die 'Cannot identify cron service; configure writer shutdown explicitly'
      if [[ " ${QUIESCE_SERVICES[*]} " != *" $cron_unit "* && " ${QUIESCE_SERVICES[*]} " != *" ${cron_unit%.service} "* ]]; then
        prompt "Nextcloud-cron gefunden. $cron_unit waehrend jeder Sicherung pausieren (betrifft auch andere Cronjobs)? [j/N]" || die 'Move the Nextcloud job to a dedicated systemd service/timer, or explicitly supply --cron-services'
        QUIESCE_SERVICES+=("$cron_unit")
      fi
    fi
  fi
  printf 'QUIESCE_SERVICES=(' >> "$UP_STATE/candidate.conf"
  # De-duplicate: a timer must precede its service so resume can reverse the order.
  for unit in "${QUIESCE_SERVICES[@]}"; do
    [[ $unit =~ ^[a-zA-Z0-9_.@:-]+$ ]] || die "Invalid service name: $unit"
  done
  printf '%s\n' "${QUIESCE_SERVICES[@]}" | python3 -c 'import sys; units=list(dict.fromkeys(s.strip() for s in sys.stdin if s.strip())); print("\n".join(sorted(units, key=lambda s: not s.endswith((".timer", ".socket")))))' > "$UP_STATE/writers"
  while IFS= read -r unit; do printf '%q ' "$unit" >> "$UP_STATE/candidate.conf"; done < "$UP_STATE/writers"
  printf ')\n' >> "$UP_STATE/candidate.conf"
  printf '%s\n' "$LOCKFILE" > "$UP_STATE/lock-path"
  # --check uses protected copies of credentials, so preparation changes no live
  # credential permissions and cannot break an older still-scheduled backup.
  cp "$UP_STATE/candidate.conf" "$UP_STATE/check.conf"
  printf '\nRCLONE_CONFIG=%q\n' "$UP_STATE/rclone.conf" >> "$UP_STATE/check.conf"
  # Load the candidate to pick up newly enabled applications and their CNFs.
  CONFIG_FILE="$UP_STATE/candidate.conf"
  load_config
  build_plan
  for i in "${!DB_CNFS[@]}"; do
    file=${DB_CNFS[i]}
    secure_file "$file"
    cp -- "$file" "$UP_STATE/db-$i.cnf"
    chmod 600 "$UP_STATE/db-$i.cnf"
    case ${DB_LABELS[i]} in db-wordpress) key=WP_DB_CNF ;; db-nextcloud) key=NC_DB_CNF ;; db-gitea) key=GITEA_DB_CNF ;; *) die 'Unknown database label' ;; esac
    printf '%s=%q\n' "$key" "$UP_STATE/db-$i.cnf" >> "$UP_STATE/check.conf"
  done
  APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-backup.sh" --config "$UP_STATE/check.conf" --check > "$UP_STATE/preflight.log" 2>&1 || {
    tail -n 40 "$UP_STATE/preflight.log"; die "Preflight failed; installed package/config/timer unchanged. Details: $UP_STATE";
  }
  sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1 > "$UP_STATE/candidate.sha256"
  echo "Vorpruefung erfolgreich. Writer: $(tr '\n' ' ' < "$UP_STATE/writers")"
  echo 'Jetzt folgen Installation, ein vollstaendiges Backup und Remote-Restore-Pruefung.'
  printf 'QUEUED\n' > "$UP_STATE/result.txt"
  if [[ $UP_FOREGROUND == true ]]; then
    flock -u 9; flock -u 8
    exec bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE"
  fi
  # A oneshot start job completes only when ExecStart exits. Waiting for that
  # job while holding FD 8 deadlocks with the worker waiting for the same lock.
  # Only enqueue here; result.txt, not systemd-run's exit code, proves success.
  # The launcher must not keep copies of our locks; systemd starts a fresh worker.
  systemd-run --no-block --unit=app-backup-upgrade --collect --property=Type=oneshot \
    --property=SELinuxContext=system_u:system_r:unconfined_service_t:s0 \
    --property=TimeoutStartSec=24h --property=TimeoutStopSec=5min --property=KillMode=mixed \
    /bin/bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" 8>&- 9>&- || {
      # Revoke the queued state while still holding the coordinator lock. Even
      # if systemd accepted the job but its reply failed, it must not install.
      printf 'FAILED: systemd submission failed; installed package/config/timer unchanged.\n' > "$UP_STATE/result.txt"
      cat "$UP_STATE/result.txt"
      echo "Details: sudo journalctl -u app-backup-upgrade.service -n 100 --no-pager"
      echo "Ergebnis: sudo cat $UP_STATE/result.txt"
      exit 1
    }
  # Release the backup lock FIRST. Once FD 8 is free, the worker may acquire
  # both immediately; reversing the order can make its backup lock check fail.
  flock -u 9
  flock -u 8
  exec 9>&- 8>&-
  echo 'Update-Auftrag an systemd uebergeben; Ausfuehrung unabhaengig von SSH.'
  echo 'Abgeschlossen ist das Update erst bei SUCCESS in der Ergebnisdatei.'
  echo 'Verlauf: sudo journalctl -fu app-backup-upgrade.service --no-pager'
  echo "Ergebnis: sudo cat $UP_STATE/result.txt"
  exit 0
fi

# -------------------- detached worker --------------------
[[ $UP_STATE == /var/backups/app-backup-upgrade-* && -d $UP_STATE && ! -L $UP_STATE && $(stat -c %u "$UP_STATE") == 0 ]] || die 'Invalid upgrade state directory'
(( (8#$(stat -c %a "$UP_STATE") & 8#077) == 0 )) || die 'Upgrade state must be private'
exec > >(tee -a "$UP_STATE/update.log") 2>&1
CONFIG_FILE=$(cat "$UP_STATE/config-path")
LOCKFILE=$(cat "$UP_STATE/lock-path")
UP_CHANGED=false
UP_OK=false
worker_finish() {
  local code=$?
  trap - EXIT INT TERM
  set +e
  if [[ $UP_OK == true && $code == 0 ]]; then
    printf 'SUCCESS\nPaket aktualisiert, Backup und Remote-Restore-Pruefung erfolgreich.\nBackup-Timer aktiv.\n' > "$UP_STATE/result.txt"
  else
    if [[ $UP_CHANGED == true ]]; then systemctl stop app-backup.timer; fi
    printf 'FAILED (exit %s)\nDetails: %s/update.log\n' "$code" "$UP_STATE" > "$UP_STATE/result.txt"
    if [[ -f $UP_STATE/rollback-update.sh ]]; then
      printf 'Rollback: sudo bash %s/rollback-update.sh\n' "$UP_STATE" >> "$UP_STATE/result.txt"
    else echo 'Installation not started; package/config unchanged.' >> "$UP_STATE/result.txt"; fi
  fi
  cat "$UP_STATE/result.txt"
  exit "$code"
}
trap worker_finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
# Record lock timeouts in update.log/result.txt as well as journald. Previously
# this wait happened before logging/traps, leaving only systemd's exit status.
flock -w 60 8 || die 'Timed out waiting for update coordinator lock (60s); installation not started'
[[ $(cat "$UP_STATE/result.txt") == QUEUED ]] || die 'Upgrade is not queued or submission was cancelled; rerun updater'
printf 'RUNNING\n' > "$UP_STATE/result.txt"
acquire_backup_lock
export APP_BACKUP_INHERIT_LOCK=1
[[ $(sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/original.sha256")" ]] || die 'Configuration changed after preparation; rerun updater'
[[ $(sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/candidate.sha256")" ]] || die 'Prepared configuration changed unexpectedly'
systemctl stop app-backup.timer
UP_CHANGED=true
# The ordinary installer records all replaced package files. Its timer remains
# disabled here until this coordinator has verified the complete new backup.
cp -- "$LIB_DIR/locking.sh" "$UP_STATE/locking.sh"
cat > "$UP_STATE/rollback-update.sh" <<'ROLLBACK'
#!/usr/bin/env bash
set -Eeuo pipefail
[[ $EUID == 0 ]] || exit 1
here=$(cd -- "$(dirname -- "$0")" && pwd)
config=$(cat "$here/config-path")
lock=$(cat "$here/lock-path")
# Package rollback acquires this same lock itself. Keep a separate coordinator
# lock across package rollback + config restoration to exclude another updater.
exec 8>/run/app-backup-upgrade.lock
flock -n 8 || { echo 'Update still active'; exit 1; }
systemctl stop app-backup.timer
LOCKFILE=$lock
source "$here/locking.sh"
acquire_backup_lock
export APP_BACKUP_INHERIT_LOCK=1
if [[ -f $here/package-rollback/rollback.sh ]]; then
  bash "$here/package-rollback/rollback.sh"
else echo 'No package rollback inventory exists; inspect installer log before proceeding.'; exit 1; fi
install -m 600 -o root -g root "$here/original.conf" "$config.rollback-tmp"
mv -fT "$config.rollback-tmp" "$config"
echo 'Package and previous application-backup configuration restored. Timer remains stopped.'
echo 'Credentials/keys were not rotated. Review services and data before restarting schedules.'
ROLLBACK
chmod 700 "$UP_STATE/rollback-update.sh"
bash "$UPDATE_ROOT/install.sh" --no-timer --snapshot-dir "$UP_STATE/package-rollback"
# Install root-owned config atomically. Never replace active DB credential values.
rclone_path=$(cat "$UP_STATE/rclone-path")
if [[ ! -e $rclone_path ]]; then
  install -m 600 -o root -g root "$UP_STATE/rclone.conf" "$rclone_path"
else
  [[ ! -L $rclone_path && $(stat -c %u "$rclone_path") == 0 ]] || die 'Unsafe live rclone configuration'
  chmod 600 "$rclone_path"
fi
install -m 600 -o root -g root "$UP_STATE/candidate.conf" "$CONFIG_FILE.update-tmp"
mv -fT "$CONFIG_FILE.update-tmp" "$CONFIG_FILE"
load_config
build_plan
for file in "${DB_CNFS[@]}"; do secure_file "$file"; chmod 600 "$file"; done
# Hold the shared lock across every child. This also excludes manual restores.
bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --check
# Disable all retention in this acceptance run, including local retention.
bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --no-retention
backup_run=$(python3 - "$WORKDIR/last-backup.json" <<'PY'
import json,sys
value=json.load(open(sys.argv[1]))
assert value['exit_code'] == 0
print(value['run'])
PY
)
bash /usr/local/sbin/app-restore.sh --config "$CONFIG_FILE" --remote-run "$backup_run" --verify-only
printf '%s\n' "$backup_run" > "$UP_STATE/verified-backup-run"
# A missed persistent timer can fire immediately, so release the shared lock first.
unset APP_BACKUP_INHERIT_LOCK
flock -u 9
systemctl enable --now app-backup.timer
UP_OK=true