johannes/app-backup
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
72969605fd39a6079ab7a47047477208a7c19ddd
app-backup/update.sh
T

453 lines
24 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# Upgrade coordinator: prepare locally, then run the long operation under systemd.
# No Git checkout, credential template replacement, or application restore needed.
set -Eeuo pipefail
umask 077
UPDATE_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
LIB_DIR="$UPDATE_ROOT/lib"
# shellcheck source=lib/common.sh
source "$LIB_DIR/common.sh"
# shellcheck source=lib/locking.sh
source "$LIB_DIR/locking.sh"
CONFIG_FILE=/etc/app-backup/app-backup.conf
UP_FOREGROUND=false
UP_INSTALL_DEPS=false
UP_GITEA=''
UP_RCLONE=''
UP_CRON=''
UP_MAIL_DIR=''
UP_MAIL_HOMES=false
UP_STATE=''
UP_WORKER=false
UP_RESTORE_TEST=false
UP_AI_REM=false
while (( $# )); do
case $1 in
--config) CONFIG_FILE=${2:?Missing configuration}; shift 2 ;;
--foreground) UP_FOREGROUND=true; shift ;;
--install-deps) UP_INSTALL_DEPS=true; shift ;;
--enable-ai-rem) UP_AI_REM=true; shift ;;
--enable-restore-test) UP_RESTORE_TEST=true; shift ;;
--enable-gitea) UP_GITEA=true; shift ;;
--no-gitea) UP_GITEA=false; shift ;;
--rclone-config) UP_RCLONE=${2:?Missing rclone configuration}; shift 2 ;;
--cron-services) UP_CRON=${2:?Missing comma-separated systemd units}; shift 2 ;;
--mail-dir) UP_MAIL_DIR=${2:?Missing mailbox directory}; shift 2 ;;
--mail-homes-only) UP_MAIL_HOMES=true; shift ;;
--worker) UP_WORKER=true; UP_STATE=${2:?Missing state}; shift 2 ;;
--help) echo 'Usage: sudo bash update.sh [--foreground] [--config FILE] [--rclone-config FILE] [--enable-gitea | --no-gitea] [--cron-services UNIT,...] [--mail-dir DIR | --mail-homes-only] [--install-deps] [--enable-restore-test] [--enable-ai-rem]'; exit 0 ;;
*) die "Unknown argument: $1" ;;
esac
done
[[ -z $UP_MAIL_DIR || $UP_MAIL_HOMES == false ]] || die 'Choose --mail-dir OR --mail-homes-only'
[[ $EUID == 0 ]] || die 'Run with sudo bash'
need systemctl; need flock
# This lock excludes two coordinators, independently from ordinary backup jobs.
exec 8>/run/app-backup-upgrade.lock
# Workers install their persistent error reporting before waiting on this lock.
if [[ $UP_WORKER == false ]]; then flock -n 8 || die 'Another updater is preparing/running'; fi
prompt() {
local answer
[[ -t 0 ]] || die "$1 (noninteractive: supply the matching command-line option/configuration)"
read -r -p "$1 " answer
[[ $answer == j || $answer == J || $answer == ja || $answer == y || $answer == yes ]]
}
append_setting() { printf '%s=%q\n' "$1" "$2" >> "$UP_STATE/candidate.conf"; }
discover() { python3 "$LIB_DIR/update-discover.py" "$@"; }
# Do not infer mailbox coverage from an existing directory. In particular,
# creating an empty /var/vmail would make preflight pass without saving mail.
prepare_mail() {
local choice home
if [[ $ENABLE_MAIL != true ]]; then
[[ -z $UP_MAIL_DIR && $UP_MAIL_HOMES == false ]] || die 'Mail options require ENABLE_MAIL=true'
return 0
fi
if [[ -n $UP_MAIL_DIR ]]; then
MAIL_DIR=$UP_MAIL_DIR
ENABLE_MAIL_STORE=true
elif [[ $UP_MAIL_HOMES == true ]]; then
ENABLE_MAIL_STORE=false
elif [[ $ENABLE_MAIL_STORE == true && ! -d $MAIL_DIR ]]; then
echo "Konfiguriertes zentrales Mailverzeichnis fehlt: $MAIL_DIR"
printf 'Konfigurierte Mail-Homes: '; printf '%q ' "${MAIL_HOME_DIRS[@]}"; echo
echo 'Absoluten Pfad zur tatsaechlichen zentralen Mailablage angeben.'
echo 'Nur wenn ALLE Mailboxen in den konfigurierten Homes liegen: homes eingeben.'
echo 'Bei unbekannter Ablage abbrechen; kein leeres Ersatzverzeichnis anlegen.'
[[ -t 0 ]] || die 'Missing mail source: supply --mail-dir DIR or explicitly confirm coverage with --mail-homes-only'
read -r -p 'Mailverzeichnis / homes / Enter zum Abbrechen: ' choice
[[ -n $choice ]] || die 'Mail storage unresolved; installed package/config/timer unchanged'
if [[ $choice == homes ]]; then
printf 'Home-Excludes: '; printf '%q ' "${MAIL_HOME_EXCLUDES[@]}"; echo
prompt 'Alle Mailboxen liegen in diesen Homes, ohne ausgeschlossene Pfade oder weitere Mounts? [j/N]' || die 'Mail coverage not confirmed'
ENABLE_MAIL_STORE=false
else MAIL_DIR=$choice; fi
fi
if [[ $ENABLE_MAIL_STORE == true ]]; then
[[ $MAIL_DIR == /* ]] || die 'MAIL_DIR must be absolute'
check_source_path "$MAIL_DIR"
append_setting MAIL_DIR "$MAIL_DIR"
echo "Zentrales Mailverzeichnis: $MAIL_DIR"
else
[[ $ENABLE_MAIL_HOME_DIRS == true && ${#MAIL_HOME_DIRS[@]} -gt 0 ]] || die 'Homes-only mail requires enabled, nonempty MAIL_HOME_DIRS'
for home in "${MAIL_HOME_DIRS[@]}"; do
[[ $home == /* ]] || die 'MAIL_HOME_DIRS must be absolute'
check_source_path "$home"
done
printf 'Mail ausschliesslich in Homes (Excludes und Mountgrenzen gelten): '
printf '%q ' "${MAIL_HOME_DIRS[@]}"; echo
fi
append_setting ENABLE_MAIL_STORE "$ENABLE_MAIL_STORE"
}
if [[ $UP_WORKER == false ]]; then
systemctl is-active --quiet app-backup-upgrade.service && die 'Upgrade service is already running'
# Only install missing, small prerequisites when requested. Never upgrade PHP,
# database server or rclone to a guessed version as a side effect of this update.
missing=()
for pair in python3:python3 rsync:rsync zstd:zstd gzip:gzip tar:tar sudo:sudo; do
tool=${pair%%:*}; package=${pair#*:}
command -v "$tool" >/dev/null || missing+=("$package")
done
if (( ${#missing[@]} )); then
if [[ $UP_INSTALL_DEPS == true ]] || prompt "Fehlende Basispakete installieren: ${missing[*]}? [j/N]"; then
need dnf
dnf install -y "${missing[@]}"
else die "Missing prerequisites: ${missing[*]}"; fi
fi
[[ $CONFIG_FILE == /etc/app-backup/app-backup.conf ]] || die 'The service uses /etc/app-backup/app-backup.conf; migrate that canonical configuration'
secure_file "$CONFIG_FILE"
UP_STATE=$(mktemp -d /var/backups/app-backup-upgrade-XXXXXXXX)
chmod 700 "$UP_STATE"
cp -a -- "$CONFIG_FILE" "$UP_STATE/original.conf"
printf '%s\n' "$CONFIG_FILE" > "$UP_STATE/config-path"
sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1 > "$UP_STATE/original.sha256"
cp -a /etc/app-backup "$UP_STATE/etc-app-backup"
echo "Update-Sicherung und Protokoll: $UP_STATE"
# Read only trusted local Bash configuration; remote metadata is never sourced.
# shellcheck disable=SC1090
source "$CONFIG_FILE"
UP_HAD_GITEA=${ENABLE_GITEA+x}
UP_ORIGINAL_RCLONE=${RCLONE_CONFIG:-}
load_config
acquire_backup_lock 1
# Preserve original text and append effective safety settings. Repeated upgrades
# may append another small block; the last explicit values remain authoritative.
cp -- "$CONFIG_FILE" "$UP_STATE/candidate.conf"
chmod 600 "$UP_STATE/candidate.conf"
printf '\n# Automated package migration (original preserved in update snapshot)\n' >> "$UP_STATE/candidate.conf"
append_setting ENABLE_REMOTE_RETENTION false
# Normal local age retention is skipped; safe space recovery remains enabled.
append_setting ENABLE_UPLOAD true
append_setting ENABLE_NEXTCLOUD_MAINTENANCE true
append_setting MAIL_BACKUP_MODE stop
if [[ $UP_RESTORE_TEST == true ]]; then
append_setting ENABLE_RESTORE_TEST true
append_setting RESTORE_TEST_MAX_WORK_GB 0
fi
prepare_mail
if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then
for pair in "ENABLE_HTTPD_STOP:$HTTPD_SERVICE_NAME" "ENABLE_PHPFPM_STOP:$PHPFPM_SERVICE_NAME"; do
name=${pair%%:*}; unit=${pair#*:}
state=$(systemctl show "$unit" --property=LoadState --value)
if [[ $state == loaded ]]; then append_setting "$name" true
elif [[ $state == not-found ]]; then append_setting "$name" false
else die "Cannot inspect web service: $unit ($state)"; fi
done
fi
if [[ -n $UP_GITEA ]]; then
ENABLE_GITEA=$UP_GITEA
elif [[ -z $UP_HAD_GITEA && $(systemctl show "$GITEA_SERVICE_NAME" --property=LoadState --value) == loaded ]]; then
if prompt 'Gitea wurde gefunden, bisher aber nicht explizit konfiguriert. In die Sicherung aufnehmen? [j/N]'; then ENABLE_GITEA=true
else die 'Choose --no-gitea explicitly if Gitea should remain excluded'; fi
fi
if [[ $ENABLE_GITEA == true && -z $UP_HAD_GITEA ]]; then
[[ -f $GITEA_ETC_DIR/app.ini ]] || die 'Gitea app.ini missing; configure Gitea paths first'
discover gitea "$GITEA_ETC_DIR/app.ini" > "$UP_STATE/gitea-paths.conf"
# Output contains ONLY allowlisted shell-quoted path/DB-name assignments.
# shellcheck disable=SC1090,SC1091
source "$UP_STATE/gitea-paths.conf"
[[ -n ${GITEA_REPOS_DIR:-} && -n ${GITEA_DB_NAME:-} ]] || die 'Gitea repository path/database not explicit; set GITEA_REPOS_DIR and GITEA_DB_NAME in the existing config'
: "${GITEA_DB_CNF:=/etc/app-backup/db-gitea.cnf}"
for name in GITEA_DATA_DIR GITEA_ETC_DIR GITEA_REPOS_DIR GITEA_DB_NAME GITEA_DB_CNF; do append_setting "$name" "${!name}"; done
fi
append_setting ENABLE_GITEA "$ENABLE_GITEA"
append_setting ENABLE_GITEA_SERVICE_STOP true
[[ $ENABLE_GITEA == true ]] || echo 'Hinweis: Gitea bleibt entsprechend der Konfiguration deaktiviert.'
# Keep keys/tokens. If a user-owned config was used before, copy it into a
# dedicated root-owned file; never chown a user's actively used configuration.
if [[ -n $UP_RCLONE ]]; then RCLONE_CONFIG=$UP_RCLONE
elif [[ -n $UP_ORIGINAL_RCLONE ]]; then RCLONE_CONFIG=$UP_ORIGINAL_RCLONE
else
detected=$(discover rclone-config)
if [[ -n $detected ]]; then RCLONE_CONFIG=$detected
else
RCLONE_CONFIG=$("$RCLONE_BIN" config file | tail -n 1)
fi
fi
if [[ ! -f $RCLONE_CONFIG || -L $RCLONE_CONFIG ]]; then
[[ -t 0 ]] || die 'rclone config not found; supply --rclone-config FILE'
read -r -p 'Pfad zur bisher verwendeten rclone.conf: ' RCLONE_CONFIG
fi
[[ -f $RCLONE_CONFIG && ! -L $RCLONE_CONFIG ]] || die 'rclone config is not a regular file'
cp -- "$RCLONE_CONFIG" "$UP_STATE/rclone.conf"
chmod 600 "$UP_STATE/rclone.conf"
# Use the original root-owned path where possible, preserving OAuth refreshes.
if [[ $(stat -c %u "$RCLONE_CONFIG") != 0 ]]; then
RCLONE_CONFIG=/etc/app-backup/rclone-migrated.conf
[[ ! -e $RCLONE_CONFIG ]] || die 'rclone-migrated.conf already exists; specify it explicitly'
fi
append_setting RCLONE_CONFIG "$RCLONE_CONFIG"
printf '%s\n' "$RCLONE_CONFIG" > "$UP_STATE/rclone-path"
if [[ $UP_AI_REM == true ]]; then
# Gemini's ai-rem is managed by Compose through ai-rem.service. Stop its
# schedulers and their jobs before the container; resume in reverse order.
# Append paths without reordering existing extra-N archive/restore labels.
for path in /srv/ai-rem /etc/ai-rem; do
check_source_path "$path"
found=false
for existing in "${EXTRA_PATHS[@]}"; do
if [[ $(realpath -m -- "$existing") == "$path" ]]; then found=true; break; fi
done
if [[ $found == false ]]; then EXTRA_PATHS+=("$path"); fi
done
{
printf 'EXTRA_PATHS=('
printf '%q ' "${EXTRA_PATHS[@]}"
printf ')\n'
} >> "$UP_STATE/candidate.conf"
# Repeated activation also restores the required stop order.
ai_rem_units=(ai-rem-healthcheck.timer ai-rem-backup.timer ai-rem-prune.timer
ai-rem-healthcheck.service ai-rem-backup.service ai-rem-prune.service ai-rem.service)
remaining=()
for unit in "${QUIESCE_SERVICES[@]}"; do
case ${unit%.service} in
ai-rem|ai-rem-healthcheck|ai-rem-backup|ai-rem-prune|ai-rem-managed-update|ai-rem-healthcheck.timer|ai-rem-backup.timer|ai-rem-prune.timer|ai-rem-managed-update.timer) ;;
*) remaining+=("$unit") ;;
esac
done
for unit in "${ai_rem_units[@]}"; do
[[ $(systemctl show "$unit" --property=LoadState --value) == loaded ]] || die "Missing ai-rem unit: $unit"
done
# Older hosts have no managed updater yet. When installed, stop its timer
# and job before taking the database snapshot; resume them after ai-rem.
optional_ai_rem_units=()
for unit in ai-rem-managed-update.timer ai-rem-managed-update.service; do
if [[ $(systemctl show "$unit" --property=LoadState --value) == loaded ]]; then
optional_ai_rem_units+=("$unit")
fi
done
ai_rem_units=("${optional_ai_rem_units[@]}" "${ai_rem_units[@]}")
QUIESCE_SERVICES=("${ai_rem_units[@]}" "${remaining[@]}")
echo 'ai-rem included: /srv/ai-rem and /etc/ai-rem; container and maintenance jobs stopped during snapshot'
fi
if [[ -n $UP_CRON ]]; then IFS=, read -r -a additional <<< "$UP_CRON"; QUIESCE_SERVICES+=("${additional[@]}"); fi
if [[ $ENABLE_NEXTCLOUD == true ]]; then
discover cron-units "$NC_DIR" > "$UP_STATE/cron-units"
mapfile -t detected_units < "$UP_STATE/cron-units"
QUIESCE_SERVICES+=("${detected_units[@]}")
# Arbitrary cron wrappers cannot be safely translated into systemd timers.
# Offer a precise policy choice rather than silently omit an active writer.
cron_result=0
discover check-crontabs "$NC_DIR" "$NC_OCC_USER" || cron_result=$?
[[ $cron_result == 0 || $cron_result == 2 ]] || die 'Cron inventory failed; no safe writer plan available'
if [[ $cron_result == 2 ]]; then
cron_unit=crond.service
if [[ $(systemctl show "$cron_unit" --property=LoadState --value) != loaded ]]; then cron_unit=cron.service; fi
[[ $(systemctl show "$cron_unit" --property=LoadState --value) == loaded ]] || die 'Cannot identify cron service; configure writer shutdown explicitly'
if [[ " ${QUIESCE_SERVICES[*]} " != *" $cron_unit "* && " ${QUIESCE_SERVICES[*]} " != *" ${cron_unit%.service} "* ]]; then
prompt "Nextcloud-cron gefunden. $cron_unit waehrend jeder Sicherung pausieren (betrifft auch andere Cronjobs)? [j/N]" || die 'Move the Nextcloud job to a dedicated systemd service/timer, or explicitly supply --cron-services'
QUIESCE_SERVICES+=("$cron_unit")
fi
fi
fi
printf 'QUIESCE_SERVICES=(' >> "$UP_STATE/candidate.conf"
# De-duplicate: a timer must precede its service so resume can reverse the order.
for unit in "${QUIESCE_SERVICES[@]}"; do
[[ $unit =~ ^[a-zA-Z0-9_.@:-]+$ ]] || die "Invalid service name: $unit"
done
printf '%s\n' "${QUIESCE_SERVICES[@]}" | python3 -c 'import sys; units=list(dict.fromkeys(s.strip() for s in sys.stdin if s.strip())); print("\n".join(sorted(units, key=lambda s: not s.endswith((".timer", ".socket")))))' > "$UP_STATE/writers"
while IFS= read -r unit; do printf '%q ' "$unit" >> "$UP_STATE/candidate.conf"; done < "$UP_STATE/writers"
printf ')\n' >> "$UP_STATE/candidate.conf"
printf '%s\n' "$LOCKFILE" > "$UP_STATE/lock-path"
# --check uses protected copies of credentials, so preparation changes no live
# credential permissions and cannot break an older still-scheduled backup.
cp "$UP_STATE/candidate.conf" "$UP_STATE/check.conf"
printf '\nRCLONE_CONFIG=%q\n' "$UP_STATE/rclone.conf" >> "$UP_STATE/check.conf"
# Load the candidate to pick up newly enabled applications and their CNFs.
CONFIG_FILE="$UP_STATE/candidate.conf"
load_config
build_plan
for i in "${!DB_CNFS[@]}"; do
file=${DB_CNFS[i]}
secure_file "$file"
cp -- "$file" "$UP_STATE/db-$i.cnf"
chmod 600 "$UP_STATE/db-$i.cnf"
case ${DB_LABELS[i]} in db-wordpress) key=WP_DB_CNF ;; db-nextcloud) key=NC_DB_CNF ;; db-gitea) key=GITEA_DB_CNF ;; *) die 'Unknown database label' ;; esac
printf '%s=%q\n' "$key" "$UP_STATE/db-$i.cnf" >> "$UP_STATE/check.conf"
done
APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-backup.sh" --config "$UP_STATE/check.conf" --prepare-space > "$UP_STATE/preflight.log" 2>&1 || {
tail -n 40 "$UP_STATE/preflight.log"; die "Preflight failed; installed package/config/timer unchanged. Details: $UP_STATE";
}
if [[ $ENABLE_RESTORE_TEST == true ]]; then
APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-restore-test.sh" --config "$UP_STATE/check.conf" --cleanup >> "$UP_STATE/preflight.log" 2>&1
APP_BACKUP_INHERIT_LOCK=1 bash "$LIB_DIR/prepare-restore-test.sh" "$UP_STATE/check.conf" >> "$UP_STATE/preflight.log" 2>&1 || {
tail -n 30 "$UP_STATE/preflight.log"; die 'Automatic MariaDB preparation failed; see preflight.log';
}
# The preparation publishes a detected image only after the isolated probe.
CONFIG_FILE="$UP_STATE/check.conf"
load_config
append_setting RESTORE_TEST_DB_IMAGE "$RESTORE_TEST_DB_IMAGE"
APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-restore-test.sh" --config "$UP_STATE/check.conf" --check >> "$UP_STATE/preflight.log" 2>&1 || {
tail -n 20 "$UP_STATE/preflight.log"; die 'Restore-test prerequisites failed; prepare local DB image and disk budget first';
}
fi
sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1 > "$UP_STATE/candidate.sha256"
echo "Vorpruefung erfolgreich. Writer: $(tr '\n' ' ' < "$UP_STATE/writers")"
echo 'Jetzt folgen Installation, ein vollstaendiges Backup und Remote-Dateipruefung.'
printf 'QUEUED\n' > "$UP_STATE/result.txt"
if [[ $UP_FOREGROUND == true ]]; then
flock -u 9; flock -u 8
exec bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE"
fi
# A oneshot start job completes only when ExecStart exits. Waiting for that
# job while holding FD 8 deadlocks with the worker waiting for the same lock.
# Only enqueue here; result.txt, not systemd-run's exit code, proves success.
# The launcher must not keep copies of our locks; systemd starts a fresh worker.
systemd-run --no-block --unit=app-backup-upgrade --collect --property=Type=oneshot \
--property=SELinuxContext=system_u:system_r:unconfined_service_t:s0 \
--property=TimeoutStartSec=24h --property=TimeoutStopSec=5min --property=KillMode=control-group \
/bin/bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" 8>&- 9>&- || {
# Revoke the queued state while still holding the coordinator lock. Even
# if systemd accepted the job but its reply failed, it must not install.
printf 'FAILED: systemd submission failed; installed package/config/timer unchanged.\n' > "$UP_STATE/result.txt"
cat "$UP_STATE/result.txt"
echo "Details: sudo journalctl -u app-backup-upgrade.service -n 100 --no-pager"
echo "Ergebnis: sudo cat $UP_STATE/result.txt"
exit 1
}
# Release the backup lock FIRST. Once FD 8 is free, the worker may acquire
# both immediately; reversing the order can make its backup lock check fail.
flock -u 9
flock -u 8
exec 9>&- 8>&-
echo 'Update-Auftrag an systemd uebergeben; Ausfuehrung unabhaengig von SSH.'
echo 'Abgeschlossen ist das Update erst bei SUCCESS in der Ergebnisdatei.'
echo 'Verlauf: sudo journalctl -fu app-backup-upgrade.service --no-pager'
echo "Ergebnis: sudo cat $UP_STATE/result.txt"
exit 0
fi
# -------------------- detached worker --------------------
[[ $UP_STATE == /var/backups/app-backup-upgrade-* && -d $UP_STATE && ! -L $UP_STATE && $(stat -c %u "$UP_STATE") == 0 ]] || die 'Invalid upgrade state directory'
(( (8#$(stat -c %a "$UP_STATE") & 8#077) == 0 )) || die 'Upgrade state must be private'
# The group receives TERM together; keep logging available for child recovery.
exec > >(trap '' INT TERM; exec tee -a "$UP_STATE/update.log") 2>&1
CONFIG_FILE=$(cat "$UP_STATE/config-path")
LOCKFILE=$(cat "$UP_STATE/lock-path")
UP_CHANGED=false
UP_OK=false
worker_finish() {
local code=$?
trap - EXIT INT TERM
set +e
if [[ $UP_OK == true && $code == 0 ]]; then
printf 'SUCCESS\nPaket aktualisiert, Backup und Remote-Dateipruefung erfolgreich.\nArchiv-Inhaltspruefung erfolgt beim monatlichen Restore-Test.\nBackup-Timer aktiv.\n' > "$UP_STATE/result.txt"
else
if [[ $UP_CHANGED == true ]]; then systemctl stop app-backup.timer app-restore-test.timer; fi
printf 'FAILED (exit %s)\nDetails: %s/update.log\n' "$code" "$UP_STATE" > "$UP_STATE/result.txt"
if [[ -f $UP_STATE/rollback-update.sh ]]; then
printf 'Rollback: sudo bash %s/rollback-update.sh\n' "$UP_STATE" >> "$UP_STATE/result.txt"
else echo 'Installation not started; package/config unchanged.' >> "$UP_STATE/result.txt"; fi
fi
cat "$UP_STATE/result.txt"
exit "$code"
}
trap worker_finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
# Record lock timeouts in update.log/result.txt as well as journald. Previously
# this wait happened before logging/traps, leaving only systemd's exit status.
flock -w 60 8 || die 'Timed out waiting for update coordinator lock (60s); installation not started'
[[ $(cat "$UP_STATE/result.txt") == QUEUED ]] || die 'Upgrade is not queued or submission was cancelled; rerun updater'
printf 'RUNNING\n' > "$UP_STATE/result.txt"
acquire_backup_lock 1
export APP_BACKUP_INHERIT_LOCK=1
[[ $(sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/original.sha256")" ]] || die 'Configuration changed after preparation; rerun updater'
[[ $(sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/candidate.sha256")" ]] || die 'Prepared configuration changed unexpectedly'
systemctl stop app-backup.timer
if systemctl is-active --quiet app-restore-test.timer; then systemctl stop app-restore-test.timer; fi
UP_CHANGED=true
# The ordinary installer records all replaced package files. Its timer remains
# disabled here until this coordinator has verified the complete new backup.
cp -- "$LIB_DIR/locking.sh" "$UP_STATE/locking.sh"
cat > "$UP_STATE/rollback-update.sh" <<'ROLLBACK'
#!/usr/bin/env bash
set -Eeuo pipefail
[[ $EUID == 0 ]] || exit 1
here=$(cd -- "$(dirname -- "$0")" && pwd)
config=$(cat "$here/config-path")
lock=$(cat "$here/lock-path")
# Package rollback acquires this same lock itself. Keep a separate coordinator
# lock across package rollback + config restoration to exclude another updater.
exec 8>/run/app-backup-upgrade.lock
flock -n 8 || { echo 'Update still active'; exit 1; }
systemctl stop app-backup.timer
LOCKFILE=$lock
source "$here/locking.sh"
acquire_backup_lock 1
export APP_BACKUP_INHERIT_LOCK=1
if [[ -f $here/package-rollback/rollback.sh ]]; then
bash "$here/package-rollback/rollback.sh"
else echo 'No package rollback inventory exists; inspect installer log before proceeding.'; exit 1; fi
install -m 600 -o root -g root "$here/original.conf" "$config.rollback-tmp"
mv -fT "$config.rollback-tmp" "$config"
echo 'Package and previous application-backup configuration restored. Timer remains stopped.'
echo 'Credentials/keys were not rotated. Review services and data before restarting schedules.'
ROLLBACK
chmod 700 "$UP_STATE/rollback-update.sh"
bash "$UPDATE_ROOT/install.sh" --no-timer --snapshot-dir "$UP_STATE/package-rollback"
# Install root-owned config atomically. Never replace active DB credential values.
rclone_path=$(cat "$UP_STATE/rclone-path")
if [[ ! -e $rclone_path ]]; then
install -m 600 -o root -g root "$UP_STATE/rclone.conf" "$rclone_path"
else
[[ ! -L $rclone_path && $(stat -c %u "$rclone_path") == 0 ]] || die 'Unsafe live rclone configuration'
chmod 600 "$rclone_path"
fi
install -m 600 -o root -g root "$UP_STATE/candidate.conf" "$CONFIG_FILE.update-tmp"
mv -fT "$CONFIG_FILE.update-tmp" "$CONFIG_FILE"
load_config
build_plan
for file in "${DB_CNFS[@]}"; do secure_file "$file"; chmod 600 "$file"; done
# Hold the shared lock across every child. This also excludes manual restores.
bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --check
# Skip normal age retention for acceptance; retain safe local space recovery.
bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --no-retention
backup_run=$(python3 - "$WORKDIR/last-backup.json" <<'PY'
import json,sys
value=json.load(open(sys.argv[1]))
assert value['exit_code'] == 0
print(value['run'])
PY
)
# The backup already confirmed remote file presence, sizes and COMPLETE.
# Archive downloads belong to the monthly rehearsal, not update acceptance.
printf '%s\n' "$backup_run" > "$UP_STATE/uploaded-backup-run"
echo '-- Update acceptance: upload and remote metadata confirmed; archive restore deferred to monthly test'
# A missed persistent timer can fire immediately, so release the shared lock first.
unset APP_BACKUP_INHERIT_LOCK
flock -u 9
systemctl enable --now app-backup.timer
if [[ $ENABLE_RESTORE_TEST == true ]]; then
systemctl enable app-restore-test-cleanup.service
systemctl enable --now app-restore-test.timer
else
systemctl disable --now app-restore-test.timer || true
fi
UP_OK=true
Reference in new issue View Git Blame Copy Permalink