#!/usr/bin/env bash # Upgrade coordinator: prepare locally, then run the long operation under systemd. # No Git checkout, credential template replacement, or application restore needed. set -Eeuo pipefail umask 077 UPDATE_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) LIB_DIR="$UPDATE_ROOT/lib" # shellcheck source=lib/common.sh source "$LIB_DIR/common.sh" # shellcheck source=lib/locking.sh source "$LIB_DIR/locking.sh" CONFIG_FILE=/etc/app-backup/app-backup.conf UP_FOREGROUND=false UP_INSTALL_DEPS=false UP_GITEA='' UP_RCLONE='' UP_CRON='' UP_STATE='' UP_WORKER=false while (( $# )); do case $1 in --config) CONFIG_FILE=${2:?Missing configuration}; shift 2 ;; --foreground) UP_FOREGROUND=true; shift ;; --install-deps) UP_INSTALL_DEPS=true; shift ;; --enable-gitea) UP_GITEA=true; shift ;; --no-gitea) UP_GITEA=false; shift ;; --rclone-config) UP_RCLONE=${2:?Missing rclone configuration}; shift 2 ;; --cron-services) UP_CRON=${2:?Missing comma-separated systemd units}; shift 2 ;; --worker) UP_WORKER=true; UP_STATE=${2:?Missing state}; shift 2 ;; --help) echo 'Usage: sudo bash update.sh [--foreground] [--config FILE] [--rclone-config FILE] [--enable-gitea | --no-gitea] [--cron-services UNIT,...] [--install-deps]'; exit 0 ;; *) die "Unknown argument: $1" ;; esac done [[ $EUID == 0 ]] || die 'Run with sudo bash' need systemctl; need flock # This lock excludes two coordinators, independently from ordinary backup jobs. exec 8>/run/app-backup-upgrade.lock if [[ $UP_WORKER == true ]]; then flock -w 60 8; else flock -n 8 || die 'Another updater is preparing/running'; fi prompt() { local answer [[ -t 0 ]] || die "$1 (noninteractive: supply the matching command-line option/configuration)" read -r -p "$1 " answer [[ $answer == j || $answer == J || $answer == ja || $answer == y || $answer == yes ]] } append_setting() { printf '%s=%q\n' "$1" "$2" >> "$UP_STATE/candidate.conf"; } discover() { python3 "$LIB_DIR/update-discover.py" "$@"; } if [[ $UP_WORKER == false ]]; then systemctl is-active --quiet app-backup-upgrade.service && die 'Upgrade service is already running' # Only install missing, small prerequisites when requested. Never upgrade PHP, # database server or rclone to a guessed version as a side effect of this update. missing=() for pair in python3:python3 rsync:rsync zstd:zstd gzip:gzip tar:tar sudo:sudo; do tool=${pair%%:*}; package=${pair#*:} command -v "$tool" >/dev/null || missing+=("$package") done if (( ${#missing[@]} )); then if [[ $UP_INSTALL_DEPS == true ]] || prompt "Fehlende Basispakete installieren: ${missing[*]}? [j/N]"; then need dnf dnf install -y "${missing[@]}" else die "Missing prerequisites: ${missing[*]}"; fi fi [[ $CONFIG_FILE == /etc/app-backup/app-backup.conf ]] || die 'The service uses /etc/app-backup/app-backup.conf; migrate that canonical configuration' secure_file "$CONFIG_FILE" UP_STATE=$(mktemp -d /var/backups/app-backup-upgrade-XXXXXXXX) chmod 700 "$UP_STATE" cp -a -- "$CONFIG_FILE" "$UP_STATE/original.conf" printf '%s\n' "$CONFIG_FILE" > "$UP_STATE/config-path" sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1 > "$UP_STATE/original.sha256" cp -a /etc/app-backup "$UP_STATE/etc-app-backup" echo "Update-Sicherung und Protokoll: $UP_STATE" # Read only trusted local Bash configuration; remote metadata is never sourced. # shellcheck disable=SC1090 source "$CONFIG_FILE" UP_HAD_GITEA=${ENABLE_GITEA+x} UP_ORIGINAL_RCLONE=${RCLONE_CONFIG:-} load_config acquire_backup_lock # Preserve original text and append effective safety settings. Repeated upgrades # may append another small block; the last explicit values remain authoritative. cp -- "$CONFIG_FILE" "$UP_STATE/candidate.conf" chmod 600 "$UP_STATE/candidate.conf" printf '\n# Automated package migration (original preserved in update snapshot)\n' >> "$UP_STATE/candidate.conf" append_setting ENABLE_REMOTE_RETENTION false # Retention of existing local sets is also skipped for the acceptance backup. append_setting ENABLE_UPLOAD true append_setting ENABLE_NEXTCLOUD_MAINTENANCE true append_setting MAIL_BACKUP_MODE stop if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then for pair in "ENABLE_HTTPD_STOP:$HTTPD_SERVICE_NAME" "ENABLE_PHPFPM_STOP:$PHPFPM_SERVICE_NAME"; do name=${pair%%:*}; unit=${pair#*:} state=$(systemctl show "$unit" --property=LoadState --value) if [[ $state == loaded ]]; then append_setting "$name" true elif [[ $state == not-found ]]; then append_setting "$name" false else die "Cannot inspect web service: $unit ($state)"; fi done fi if [[ -n $UP_GITEA ]]; then ENABLE_GITEA=$UP_GITEA elif [[ -z $UP_HAD_GITEA && $(systemctl show "$GITEA_SERVICE_NAME" --property=LoadState --value) == loaded ]]; then if prompt 'Gitea wurde gefunden, bisher aber nicht explizit konfiguriert. In die Sicherung aufnehmen? [j/N]'; then ENABLE_GITEA=true else die 'Choose --no-gitea explicitly if Gitea should remain excluded'; fi fi if [[ $ENABLE_GITEA == true && -z $UP_HAD_GITEA ]]; then [[ -f $GITEA_ETC_DIR/app.ini ]] || die 'Gitea app.ini missing; configure Gitea paths first' discover gitea "$GITEA_ETC_DIR/app.ini" > "$UP_STATE/gitea-paths.conf" # Output contains ONLY allowlisted shell-quoted path/DB-name assignments. # shellcheck disable=SC1090,SC1091 source "$UP_STATE/gitea-paths.conf" [[ -n ${GITEA_REPOS_DIR:-} && -n ${GITEA_DB_NAME:-} ]] || die 'Gitea repository path/database not explicit; set GITEA_REPOS_DIR and GITEA_DB_NAME in the existing config' : "${GITEA_DB_CNF:=/etc/app-backup/db-gitea.cnf}" for name in GITEA_DATA_DIR GITEA_ETC_DIR GITEA_REPOS_DIR GITEA_DB_NAME GITEA_DB_CNF; do append_setting "$name" "${!name}"; done fi append_setting ENABLE_GITEA "$ENABLE_GITEA" append_setting ENABLE_GITEA_SERVICE_STOP true [[ $ENABLE_GITEA == true ]] || echo 'Hinweis: Gitea bleibt entsprechend der Konfiguration deaktiviert.' # Keep keys/tokens. If a user-owned config was used before, copy it into a # dedicated root-owned file; never chown a user's actively used configuration. if [[ -n $UP_RCLONE ]]; then RCLONE_CONFIG=$UP_RCLONE elif [[ -n $UP_ORIGINAL_RCLONE ]]; then RCLONE_CONFIG=$UP_ORIGINAL_RCLONE else detected=$(discover rclone-config) if [[ -n $detected ]]; then RCLONE_CONFIG=$detected else RCLONE_CONFIG=$("$RCLONE_BIN" config file | tail -n 1) fi fi if [[ ! -f $RCLONE_CONFIG || -L $RCLONE_CONFIG ]]; then [[ -t 0 ]] || die 'rclone config not found; supply --rclone-config FILE' read -r -p 'Pfad zur bisher verwendeten rclone.conf: ' RCLONE_CONFIG fi [[ -f $RCLONE_CONFIG && ! -L $RCLONE_CONFIG ]] || die 'rclone config is not a regular file' cp -- "$RCLONE_CONFIG" "$UP_STATE/rclone.conf" chmod 600 "$UP_STATE/rclone.conf" # Use the original root-owned path where possible, preserving OAuth refreshes. if [[ $(stat -c %u "$RCLONE_CONFIG") != 0 ]]; then RCLONE_CONFIG=/etc/app-backup/rclone-migrated.conf [[ ! -e $RCLONE_CONFIG ]] || die 'rclone-migrated.conf already exists; specify it explicitly' fi append_setting RCLONE_CONFIG "$RCLONE_CONFIG" printf '%s\n' "$RCLONE_CONFIG" > "$UP_STATE/rclone-path" if [[ -n $UP_CRON ]]; then IFS=, read -r -a additional <<< "$UP_CRON"; QUIESCE_SERVICES+=("${additional[@]}"); fi if [[ $ENABLE_NEXTCLOUD == true ]]; then discover cron-units "$NC_DIR" > "$UP_STATE/cron-units" mapfile -t detected_units < "$UP_STATE/cron-units" QUIESCE_SERVICES+=("${detected_units[@]}") # Arbitrary cron wrappers cannot be safely translated into systemd timers. # Offer a precise policy choice rather than silently omit an active writer. cron_result=0 discover check-crontabs "$NC_DIR" "$NC_OCC_USER" || cron_result=$? [[ $cron_result == 0 || $cron_result == 2 ]] || die 'Cron inventory failed; no safe writer plan available' if [[ $cron_result == 2 ]]; then cron_unit=crond.service if [[ $(systemctl show "$cron_unit" --property=LoadState --value) != loaded ]]; then cron_unit=cron.service; fi [[ $(systemctl show "$cron_unit" --property=LoadState --value) == loaded ]] || die 'Cannot identify cron service; configure writer shutdown explicitly' if [[ " ${QUIESCE_SERVICES[*]} " != *" $cron_unit "* && " ${QUIESCE_SERVICES[*]} " != *" ${cron_unit%.service} "* ]]; then prompt "Nextcloud-cron gefunden. $cron_unit waehrend jeder Sicherung pausieren (betrifft auch andere Cronjobs)? [j/N]" || die 'Move the Nextcloud job to a dedicated systemd service/timer, or explicitly supply --cron-services' QUIESCE_SERVICES+=("$cron_unit") fi fi fi printf 'QUIESCE_SERVICES=(' >> "$UP_STATE/candidate.conf" # De-duplicate: a timer must precede its service so resume can reverse the order. for unit in "${QUIESCE_SERVICES[@]}"; do [[ $unit =~ ^[a-zA-Z0-9_.@:-]+$ ]] || die "Invalid service name: $unit" done printf '%s\n' "${QUIESCE_SERVICES[@]}" | python3 -c 'import sys; units=list(dict.fromkeys(s.strip() for s in sys.stdin if s.strip())); print("\n".join(sorted(units, key=lambda s: not s.endswith((".timer", ".socket")))))' > "$UP_STATE/writers" while IFS= read -r unit; do printf '%q ' "$unit" >> "$UP_STATE/candidate.conf"; done < "$UP_STATE/writers" printf ')\n' >> "$UP_STATE/candidate.conf" printf '%s\n' "$LOCKFILE" > "$UP_STATE/lock-path" # --check uses protected copies of credentials, so preparation changes no live # credential permissions and cannot break an older still-scheduled backup. cp "$UP_STATE/candidate.conf" "$UP_STATE/check.conf" printf '\nRCLONE_CONFIG=%q\n' "$UP_STATE/rclone.conf" >> "$UP_STATE/check.conf" # Load the candidate to pick up newly enabled applications and their CNFs. CONFIG_FILE="$UP_STATE/candidate.conf" load_config build_plan for i in "${!DB_CNFS[@]}"; do file=${DB_CNFS[i]} secure_file "$file" cp -- "$file" "$UP_STATE/db-$i.cnf" chmod 600 "$UP_STATE/db-$i.cnf" case ${DB_LABELS[i]} in db-wordpress) key=WP_DB_CNF ;; db-nextcloud) key=NC_DB_CNF ;; db-gitea) key=GITEA_DB_CNF ;; *) die 'Unknown database label' ;; esac printf '%s=%q\n' "$key" "$UP_STATE/db-$i.cnf" >> "$UP_STATE/check.conf" done APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-backup.sh" --config "$UP_STATE/check.conf" --check > "$UP_STATE/preflight.log" 2>&1 || { tail -n 40 "$UP_STATE/preflight.log"; die "Preflight failed; installed package/config/timer unchanged. Details: $UP_STATE"; } sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1 > "$UP_STATE/candidate.sha256" echo "Vorpruefung erfolgreich. Writer: $(tr '\n' ' ' < "$UP_STATE/writers")" echo 'Jetzt folgen Installation, ein vollstaendiges Backup und Remote-Restore-Pruefung.' if [[ $UP_FOREGROUND == true ]]; then flock -u 9; flock -u 8 exec bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" fi # A new systemd service inherits no shell FDs; it acquires both locks itself. printf 'QUEUED\n' > "$UP_STATE/result.txt" systemd-run --unit=app-backup-upgrade --collect --property=Type=oneshot \ --property=TimeoutStartSec=24h --property=TimeoutStopSec=5min --property=KillMode=mixed \ /bin/bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" || { printf 'FAILED: systemd worker could not be started; installed package/config/timer unchanged.\n' > "$UP_STATE/result.txt" exit 1 } echo 'Update laeuft unabhaengig von SSH weiter.' echo 'Verlauf: sudo journalctl -fu app-backup-upgrade.service' echo "Ergebnis: sudo cat $UP_STATE/result.txt" exit 0 fi # -------------------- detached worker -------------------- [[ $UP_STATE == /var/backups/app-backup-upgrade-* && -d $UP_STATE && ! -L $UP_STATE && $(stat -c %u "$UP_STATE") == 0 ]] || die 'Invalid upgrade state directory' (( (8#$(stat -c %a "$UP_STATE") & 8#077) == 0 )) || die 'Upgrade state must be private' exec > >(tee -a "$UP_STATE/update.log") 2>&1 CONFIG_FILE=$(cat "$UP_STATE/config-path") LOCKFILE=$(cat "$UP_STATE/lock-path") UP_CHANGED=false UP_OK=false worker_finish() { local code=$? trap - EXIT INT TERM set +e if [[ $UP_OK == true && $code == 0 ]]; then printf 'SUCCESS\nPaket aktualisiert, Backup und Remote-Restore-Pruefung erfolgreich.\nBackup-Timer aktiv.\n' > "$UP_STATE/result.txt" else if [[ $UP_CHANGED == true ]]; then systemctl stop app-backup.timer; fi printf 'FAILED (exit %s)\nDetails: %s/update.log\n' "$code" "$UP_STATE" > "$UP_STATE/result.txt" if [[ -f $UP_STATE/rollback-update.sh ]]; then printf 'Rollback: sudo bash %s/rollback-update.sh\n' "$UP_STATE" >> "$UP_STATE/result.txt" else echo 'Installation not started; package/config unchanged.' >> "$UP_STATE/result.txt"; fi fi cat "$UP_STATE/result.txt" exit "$code" } trap worker_finish EXIT trap 'exit 130' INT trap 'exit 143' TERM printf 'RUNNING\n' > "$UP_STATE/result.txt" acquire_backup_lock export APP_BACKUP_INHERIT_LOCK=1 [[ $(sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/original.sha256")" ]] || die 'Configuration changed after preparation; rerun updater' [[ $(sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/candidate.sha256")" ]] || die 'Prepared configuration changed unexpectedly' systemctl stop app-backup.timer UP_CHANGED=true # The ordinary installer records all replaced package files. Its timer remains # disabled here until this coordinator has verified the complete new backup. cp -- "$LIB_DIR/locking.sh" "$UP_STATE/locking.sh" cat > "$UP_STATE/rollback-update.sh" <<'ROLLBACK' #!/usr/bin/env bash set -Eeuo pipefail [[ $EUID == 0 ]] || exit 1 here=$(cd -- "$(dirname -- "$0")" && pwd) config=$(cat "$here/config-path") lock=$(cat "$here/lock-path") # Package rollback acquires this same lock itself. Keep a separate coordinator # lock across package rollback + config restoration to exclude another updater. exec 8>/run/app-backup-upgrade.lock flock -n 8 || { echo 'Update still active'; exit 1; } systemctl stop app-backup.timer LOCKFILE=$lock source "$here/locking.sh" acquire_backup_lock export APP_BACKUP_INHERIT_LOCK=1 if [[ -f $here/package-rollback/rollback.sh ]]; then bash "$here/package-rollback/rollback.sh" else echo 'No package rollback inventory exists; inspect installer log before proceeding.'; exit 1; fi install -m 600 -o root -g root "$here/original.conf" "$config.rollback-tmp" mv -fT "$config.rollback-tmp" "$config" echo 'Package and previous application-backup configuration restored. Timer remains stopped.' echo 'Credentials/keys were not rotated. Review services and data before restarting schedules.' ROLLBACK chmod 700 "$UP_STATE/rollback-update.sh" bash "$UPDATE_ROOT/install.sh" --no-timer --snapshot-dir "$UP_STATE/package-rollback" # Install root-owned config atomically. Never replace active DB credential values. rclone_path=$(cat "$UP_STATE/rclone-path") if [[ ! -e $rclone_path ]]; then install -m 600 -o root -g root "$UP_STATE/rclone.conf" "$rclone_path" else [[ ! -L $rclone_path && $(stat -c %u "$rclone_path") == 0 ]] || die 'Unsafe live rclone configuration' chmod 600 "$rclone_path" fi install -m 600 -o root -g root "$UP_STATE/candidate.conf" "$CONFIG_FILE.update-tmp" mv -fT "$CONFIG_FILE.update-tmp" "$CONFIG_FILE" load_config build_plan for file in "${DB_CNFS[@]}"; do secure_file "$file"; chmod 600 "$file"; done # Hold the shared lock across every child. This also excludes manual restores. bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --check # Disable all retention in this acceptance run, including local retention. bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --no-retention backup_run=$(python3 - "$WORKDIR/last-backup.json" <<'PY' import json,sys value=json.load(open(sys.argv[1])) assert value['exit_code'] == 0 print(value['run']) PY ) bash /usr/local/sbin/app-restore.sh --config "$CONFIG_FILE" --remote-run "$backup_run" --verify-only printf '%s\n' "$backup_run" > "$UP_STATE/verified-backup-run" # A missed persistent timer can fire immediately, so release the shared lock first. unset APP_BACKUP_INHERIT_LOCK flock -u 9 systemctl enable --now app-backup.timer UP_OK=true