Add self-contained automated server upgrade package
This commit is contained in:
admin committed
1 parent
4d58fc7b66
commit
8e6c9a058f
14 files changed
+908
-26
No files matched your search
@@ -0,0 +1,307 @@
|
||||
#!/usr/bin/env bash
|
||||
# Upgrade coordinator: prepare locally, then run the long operation under systemd.
|
||||
# No Git checkout, credential template replacement, or application restore needed.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
UPDATE_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
LIB_DIR="$UPDATE_ROOT/lib"
|
||||
# shellcheck source=lib/common.sh
|
||||
source "$LIB_DIR/common.sh"
|
||||
# shellcheck source=lib/locking.sh
|
||||
source "$LIB_DIR/locking.sh"
|
||||
CONFIG_FILE=/etc/app-backup/app-backup.conf
|
||||
UP_FOREGROUND=false
|
||||
UP_INSTALL_DEPS=false
|
||||
UP_GITEA=''
|
||||
UP_RCLONE=''
|
||||
UP_CRON=''
|
||||
UP_STATE=''
|
||||
UP_WORKER=false
|
||||
while (( $# )); do
|
||||
case $1 in
|
||||
--config) CONFIG_FILE=${2:?Missing configuration}; shift 2 ;;
|
||||
--foreground) UP_FOREGROUND=true; shift ;;
|
||||
--install-deps) UP_INSTALL_DEPS=true; shift ;;
|
||||
--enable-gitea) UP_GITEA=true; shift ;;
|
||||
--no-gitea) UP_GITEA=false; shift ;;
|
||||
--rclone-config) UP_RCLONE=${2:?Missing rclone configuration}; shift 2 ;;
|
||||
--cron-services) UP_CRON=${2:?Missing comma-separated systemd units}; shift 2 ;;
|
||||
--worker) UP_WORKER=true; UP_STATE=${2:?Missing state}; shift 2 ;;
|
||||
--help) echo 'Usage: sudo bash update.sh [--foreground] [--config FILE] [--rclone-config FILE] [--enable-gitea | --no-gitea] [--cron-services UNIT,...] [--install-deps]'; exit 0 ;;
|
||||
*) die "Unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
[[ $EUID == 0 ]] || die 'Run with sudo bash'
|
||||
need systemctl; need flock
|
||||
# This lock excludes two coordinators, independently from ordinary backup jobs.
|
||||
exec 8>/run/app-backup-upgrade.lock
|
||||
if [[ $UP_WORKER == true ]]; then flock -w 60 8; else flock -n 8 || die 'Another updater is preparing/running'; fi
|
||||
|
||||
prompt() {
|
||||
local answer
|
||||
[[ -t 0 ]] || die "$1 (noninteractive: supply the matching command-line option/configuration)"
|
||||
read -r -p "$1 " answer
|
||||
[[ $answer == j || $answer == J || $answer == ja || $answer == y || $answer == yes ]]
|
||||
}
|
||||
append_setting() { printf '%s=%q\n' "$1" "$2" >> "$UP_STATE/candidate.conf"; }
|
||||
discover() { python3 "$LIB_DIR/update-discover.py" "$@"; }
|
||||
|
||||
if [[ $UP_WORKER == false ]]; then
|
||||
systemctl is-active --quiet app-backup-upgrade.service && die 'Upgrade service is already running'
|
||||
# Only install missing, small prerequisites when requested. Never upgrade PHP,
|
||||
# database server or rclone to a guessed version as a side effect of this update.
|
||||
missing=()
|
||||
for pair in python3:python3 rsync:rsync zstd:zstd gzip:gzip tar:tar sudo:sudo; do
|
||||
tool=${pair%%:*}; package=${pair#*:}
|
||||
command -v "$tool" >/dev/null || missing+=("$package")
|
||||
done
|
||||
if (( ${#missing[@]} )); then
|
||||
if [[ $UP_INSTALL_DEPS == true ]] || prompt "Fehlende Basispakete installieren: ${missing[*]}? [j/N]"; then
|
||||
need dnf
|
||||
dnf install -y "${missing[@]}"
|
||||
else die "Missing prerequisites: ${missing[*]}"; fi
|
||||
fi
|
||||
[[ $CONFIG_FILE == /etc/app-backup/app-backup.conf ]] || die 'The service uses /etc/app-backup/app-backup.conf; migrate that canonical configuration'
|
||||
secure_file "$CONFIG_FILE"
|
||||
UP_STATE=$(mktemp -d /var/backups/app-backup-upgrade-XXXXXXXX)
|
||||
chmod 700 "$UP_STATE"
|
||||
cp -a -- "$CONFIG_FILE" "$UP_STATE/original.conf"
|
||||
printf '%s\n' "$CONFIG_FILE" > "$UP_STATE/config-path"
|
||||
sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1 > "$UP_STATE/original.sha256"
|
||||
cp -a /etc/app-backup "$UP_STATE/etc-app-backup"
|
||||
echo "Update-Sicherung und Protokoll: $UP_STATE"
|
||||
# Read only trusted local Bash configuration; remote metadata is never sourced.
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
UP_HAD_GITEA=${ENABLE_GITEA+x}
|
||||
UP_ORIGINAL_RCLONE=${RCLONE_CONFIG:-}
|
||||
load_config
|
||||
acquire_backup_lock
|
||||
# Preserve original text and append effective safety settings. Repeated upgrades
|
||||
# may append another small block; the last explicit values remain authoritative.
|
||||
cp -- "$CONFIG_FILE" "$UP_STATE/candidate.conf"
|
||||
chmod 600 "$UP_STATE/candidate.conf"
|
||||
printf '\n# Automated package migration (original preserved in update snapshot)\n' >> "$UP_STATE/candidate.conf"
|
||||
append_setting ENABLE_REMOTE_RETENTION false
|
||||
# Retention of existing local sets is also skipped for the acceptance backup.
|
||||
append_setting ENABLE_UPLOAD true
|
||||
append_setting ENABLE_NEXTCLOUD_MAINTENANCE true
|
||||
append_setting MAIL_BACKUP_MODE stop
|
||||
if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then
|
||||
for pair in "ENABLE_HTTPD_STOP:$HTTPD_SERVICE_NAME" "ENABLE_PHPFPM_STOP:$PHPFPM_SERVICE_NAME"; do
|
||||
name=${pair%%:*}; unit=${pair#*:}
|
||||
state=$(systemctl show "$unit" --property=LoadState --value)
|
||||
if [[ $state == loaded ]]; then append_setting "$name" true
|
||||
elif [[ $state == not-found ]]; then append_setting "$name" false
|
||||
else die "Cannot inspect web service: $unit ($state)"; fi
|
||||
done
|
||||
fi
|
||||
if [[ -n $UP_GITEA ]]; then
|
||||
ENABLE_GITEA=$UP_GITEA
|
||||
elif [[ -z $UP_HAD_GITEA && $(systemctl show "$GITEA_SERVICE_NAME" --property=LoadState --value) == loaded ]]; then
|
||||
if prompt 'Gitea wurde gefunden, bisher aber nicht explizit konfiguriert. In die Sicherung aufnehmen? [j/N]'; then ENABLE_GITEA=true
|
||||
else die 'Choose --no-gitea explicitly if Gitea should remain excluded'; fi
|
||||
fi
|
||||
if [[ $ENABLE_GITEA == true && -z $UP_HAD_GITEA ]]; then
|
||||
[[ -f $GITEA_ETC_DIR/app.ini ]] || die 'Gitea app.ini missing; configure Gitea paths first'
|
||||
discover gitea "$GITEA_ETC_DIR/app.ini" > "$UP_STATE/gitea-paths.conf"
|
||||
# Output contains ONLY allowlisted shell-quoted path/DB-name assignments.
|
||||
# shellcheck disable=SC1090,SC1091
|
||||
source "$UP_STATE/gitea-paths.conf"
|
||||
[[ -n ${GITEA_REPOS_DIR:-} && -n ${GITEA_DB_NAME:-} ]] || die 'Gitea repository path/database not explicit; set GITEA_REPOS_DIR and GITEA_DB_NAME in the existing config'
|
||||
: "${GITEA_DB_CNF:=/etc/app-backup/db-gitea.cnf}"
|
||||
for name in GITEA_DATA_DIR GITEA_ETC_DIR GITEA_REPOS_DIR GITEA_DB_NAME GITEA_DB_CNF; do append_setting "$name" "${!name}"; done
|
||||
fi
|
||||
append_setting ENABLE_GITEA "$ENABLE_GITEA"
|
||||
append_setting ENABLE_GITEA_SERVICE_STOP true
|
||||
[[ $ENABLE_GITEA == true ]] || echo 'Hinweis: Gitea bleibt entsprechend der Konfiguration deaktiviert.'
|
||||
|
||||
# Keep keys/tokens. If a user-owned config was used before, copy it into a
|
||||
# dedicated root-owned file; never chown a user's actively used configuration.
|
||||
if [[ -n $UP_RCLONE ]]; then RCLONE_CONFIG=$UP_RCLONE
|
||||
elif [[ -n $UP_ORIGINAL_RCLONE ]]; then RCLONE_CONFIG=$UP_ORIGINAL_RCLONE
|
||||
else
|
||||
detected=$(discover rclone-config)
|
||||
if [[ -n $detected ]]; then RCLONE_CONFIG=$detected
|
||||
else
|
||||
RCLONE_CONFIG=$("$RCLONE_BIN" config file | tail -n 1)
|
||||
fi
|
||||
fi
|
||||
if [[ ! -f $RCLONE_CONFIG || -L $RCLONE_CONFIG ]]; then
|
||||
[[ -t 0 ]] || die 'rclone config not found; supply --rclone-config FILE'
|
||||
read -r -p 'Pfad zur bisher verwendeten rclone.conf: ' RCLONE_CONFIG
|
||||
fi
|
||||
[[ -f $RCLONE_CONFIG && ! -L $RCLONE_CONFIG ]] || die 'rclone config is not a regular file'
|
||||
cp -- "$RCLONE_CONFIG" "$UP_STATE/rclone.conf"
|
||||
chmod 600 "$UP_STATE/rclone.conf"
|
||||
# Use the original root-owned path where possible, preserving OAuth refreshes.
|
||||
if [[ $(stat -c %u "$RCLONE_CONFIG") != 0 ]]; then
|
||||
RCLONE_CONFIG=/etc/app-backup/rclone-migrated.conf
|
||||
[[ ! -e $RCLONE_CONFIG ]] || die 'rclone-migrated.conf already exists; specify it explicitly'
|
||||
fi
|
||||
append_setting RCLONE_CONFIG "$RCLONE_CONFIG"
|
||||
printf '%s\n' "$RCLONE_CONFIG" > "$UP_STATE/rclone-path"
|
||||
|
||||
if [[ -n $UP_CRON ]]; then IFS=, read -r -a additional <<< "$UP_CRON"; QUIESCE_SERVICES+=("${additional[@]}"); fi
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then
|
||||
discover cron-units "$NC_DIR" > "$UP_STATE/cron-units"
|
||||
mapfile -t detected_units < "$UP_STATE/cron-units"
|
||||
QUIESCE_SERVICES+=("${detected_units[@]}")
|
||||
# Arbitrary cron wrappers cannot be safely translated into systemd timers.
|
||||
# Offer a precise policy choice rather than silently omit an active writer.
|
||||
cron_result=0
|
||||
discover check-crontabs "$NC_DIR" "$NC_OCC_USER" || cron_result=$?
|
||||
[[ $cron_result == 0 || $cron_result == 2 ]] || die 'Cron inventory failed; no safe writer plan available'
|
||||
if [[ $cron_result == 2 ]]; then
|
||||
cron_unit=crond.service
|
||||
if [[ $(systemctl show "$cron_unit" --property=LoadState --value) != loaded ]]; then cron_unit=cron.service; fi
|
||||
[[ $(systemctl show "$cron_unit" --property=LoadState --value) == loaded ]] || die 'Cannot identify cron service; configure writer shutdown explicitly'
|
||||
if [[ " ${QUIESCE_SERVICES[*]} " != *" $cron_unit "* && " ${QUIESCE_SERVICES[*]} " != *" ${cron_unit%.service} "* ]]; then
|
||||
prompt "Nextcloud-cron gefunden. $cron_unit waehrend jeder Sicherung pausieren (betrifft auch andere Cronjobs)? [j/N]" || die 'Move the Nextcloud job to a dedicated systemd service/timer, or explicitly supply --cron-services'
|
||||
QUIESCE_SERVICES+=("$cron_unit")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
printf 'QUIESCE_SERVICES=(' >> "$UP_STATE/candidate.conf"
|
||||
# De-duplicate: a timer must precede its service so resume can reverse the order.
|
||||
for unit in "${QUIESCE_SERVICES[@]}"; do
|
||||
[[ $unit =~ ^[a-zA-Z0-9_.@:-]+$ ]] || die "Invalid service name: $unit"
|
||||
done
|
||||
printf '%s\n' "${QUIESCE_SERVICES[@]}" | python3 -c 'import sys; units=list(dict.fromkeys(s.strip() for s in sys.stdin if s.strip())); print("\n".join(sorted(units, key=lambda s: not s.endswith((".timer", ".socket")))))' > "$UP_STATE/writers"
|
||||
while IFS= read -r unit; do printf '%q ' "$unit" >> "$UP_STATE/candidate.conf"; done < "$UP_STATE/writers"
|
||||
printf ')\n' >> "$UP_STATE/candidate.conf"
|
||||
printf '%s\n' "$LOCKFILE" > "$UP_STATE/lock-path"
|
||||
# --check uses protected copies of credentials, so preparation changes no live
|
||||
# credential permissions and cannot break an older still-scheduled backup.
|
||||
cp "$UP_STATE/candidate.conf" "$UP_STATE/check.conf"
|
||||
printf '\nRCLONE_CONFIG=%q\n' "$UP_STATE/rclone.conf" >> "$UP_STATE/check.conf"
|
||||
# Load the candidate to pick up newly enabled applications and their CNFs.
|
||||
CONFIG_FILE="$UP_STATE/candidate.conf"
|
||||
load_config
|
||||
build_plan
|
||||
for i in "${!DB_CNFS[@]}"; do
|
||||
file=${DB_CNFS[i]}
|
||||
secure_file "$file"
|
||||
cp -- "$file" "$UP_STATE/db-$i.cnf"
|
||||
chmod 600 "$UP_STATE/db-$i.cnf"
|
||||
case ${DB_LABELS[i]} in db-wordpress) key=WP_DB_CNF ;; db-nextcloud) key=NC_DB_CNF ;; db-gitea) key=GITEA_DB_CNF ;; *) die 'Unknown database label' ;; esac
|
||||
printf '%s=%q\n' "$key" "$UP_STATE/db-$i.cnf" >> "$UP_STATE/check.conf"
|
||||
done
|
||||
APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-backup.sh" --config "$UP_STATE/check.conf" --check > "$UP_STATE/preflight.log" 2>&1 || {
|
||||
tail -n 40 "$UP_STATE/preflight.log"; die "Preflight failed; installed package/config/timer unchanged. Details: $UP_STATE";
|
||||
}
|
||||
sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1 > "$UP_STATE/candidate.sha256"
|
||||
echo "Vorpruefung erfolgreich. Writer: $(tr '\n' ' ' < "$UP_STATE/writers")"
|
||||
echo 'Jetzt folgen Installation, ein vollstaendiges Backup und Remote-Restore-Pruefung.'
|
||||
if [[ $UP_FOREGROUND == true ]]; then
|
||||
flock -u 9; flock -u 8
|
||||
exec bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE"
|
||||
fi
|
||||
# A new systemd service inherits no shell FDs; it acquires both locks itself.
|
||||
printf 'QUEUED\n' > "$UP_STATE/result.txt"
|
||||
systemd-run --unit=app-backup-upgrade --collect --property=Type=oneshot \
|
||||
--property=TimeoutStartSec=24h --property=TimeoutStopSec=5min --property=KillMode=mixed \
|
||||
/bin/bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" || {
|
||||
printf 'FAILED: systemd worker could not be started; installed package/config/timer unchanged.\n' > "$UP_STATE/result.txt"
|
||||
exit 1
|
||||
}
|
||||
echo 'Update laeuft unabhaengig von SSH weiter.'
|
||||
echo 'Verlauf: sudo journalctl -fu app-backup-upgrade.service'
|
||||
echo "Ergebnis: sudo cat $UP_STATE/result.txt"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -------------------- detached worker --------------------
|
||||
[[ $UP_STATE == /var/backups/app-backup-upgrade-* && -d $UP_STATE && ! -L $UP_STATE && $(stat -c %u "$UP_STATE") == 0 ]] || die 'Invalid upgrade state directory'
|
||||
(( (8#$(stat -c %a "$UP_STATE") & 8#077) == 0 )) || die 'Upgrade state must be private'
|
||||
exec > >(tee -a "$UP_STATE/update.log") 2>&1
|
||||
CONFIG_FILE=$(cat "$UP_STATE/config-path")
|
||||
LOCKFILE=$(cat "$UP_STATE/lock-path")
|
||||
UP_CHANGED=false
|
||||
UP_OK=false
|
||||
worker_finish() {
|
||||
local code=$?
|
||||
trap - EXIT INT TERM
|
||||
set +e
|
||||
if [[ $UP_OK == true && $code == 0 ]]; then
|
||||
printf 'SUCCESS\nPaket aktualisiert, Backup und Remote-Restore-Pruefung erfolgreich.\nBackup-Timer aktiv.\n' > "$UP_STATE/result.txt"
|
||||
else
|
||||
if [[ $UP_CHANGED == true ]]; then systemctl stop app-backup.timer; fi
|
||||
printf 'FAILED (exit %s)\nDetails: %s/update.log\n' "$code" "$UP_STATE" > "$UP_STATE/result.txt"
|
||||
if [[ -f $UP_STATE/rollback-update.sh ]]; then
|
||||
printf 'Rollback: sudo bash %s/rollback-update.sh\n' "$UP_STATE" >> "$UP_STATE/result.txt"
|
||||
else echo 'Installation not started; package/config unchanged.' >> "$UP_STATE/result.txt"; fi
|
||||
fi
|
||||
cat "$UP_STATE/result.txt"
|
||||
exit "$code"
|
||||
}
|
||||
trap worker_finish EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
printf 'RUNNING\n' > "$UP_STATE/result.txt"
|
||||
acquire_backup_lock
|
||||
export APP_BACKUP_INHERIT_LOCK=1
|
||||
[[ $(sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/original.sha256")" ]] || die 'Configuration changed after preparation; rerun updater'
|
||||
[[ $(sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/candidate.sha256")" ]] || die 'Prepared configuration changed unexpectedly'
|
||||
systemctl stop app-backup.timer
|
||||
UP_CHANGED=true
|
||||
# The ordinary installer records all replaced package files. Its timer remains
|
||||
# disabled here until this coordinator has verified the complete new backup.
|
||||
cp -- "$LIB_DIR/locking.sh" "$UP_STATE/locking.sh"
|
||||
cat > "$UP_STATE/rollback-update.sh" <<'ROLLBACK'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
[[ $EUID == 0 ]] || exit 1
|
||||
here=$(cd -- "$(dirname -- "$0")" && pwd)
|
||||
config=$(cat "$here/config-path")
|
||||
lock=$(cat "$here/lock-path")
|
||||
# Package rollback acquires this same lock itself. Keep a separate coordinator
|
||||
# lock across package rollback + config restoration to exclude another updater.
|
||||
exec 8>/run/app-backup-upgrade.lock
|
||||
flock -n 8 || { echo 'Update still active'; exit 1; }
|
||||
systemctl stop app-backup.timer
|
||||
LOCKFILE=$lock
|
||||
source "$here/locking.sh"
|
||||
acquire_backup_lock
|
||||
export APP_BACKUP_INHERIT_LOCK=1
|
||||
if [[ -f $here/package-rollback/rollback.sh ]]; then
|
||||
bash "$here/package-rollback/rollback.sh"
|
||||
else echo 'No package rollback inventory exists; inspect installer log before proceeding.'; exit 1; fi
|
||||
install -m 600 -o root -g root "$here/original.conf" "$config.rollback-tmp"
|
||||
mv -fT "$config.rollback-tmp" "$config"
|
||||
echo 'Package and previous application-backup configuration restored. Timer remains stopped.'
|
||||
echo 'Credentials/keys were not rotated. Review services and data before restarting schedules.'
|
||||
ROLLBACK
|
||||
chmod 700 "$UP_STATE/rollback-update.sh"
|
||||
bash "$UPDATE_ROOT/install.sh" --no-timer --snapshot-dir "$UP_STATE/package-rollback"
|
||||
# Install root-owned config atomically. Never replace active DB credential values.
|
||||
rclone_path=$(cat "$UP_STATE/rclone-path")
|
||||
if [[ ! -e $rclone_path ]]; then
|
||||
install -m 600 -o root -g root "$UP_STATE/rclone.conf" "$rclone_path"
|
||||
else
|
||||
[[ ! -L $rclone_path && $(stat -c %u "$rclone_path") == 0 ]] || die 'Unsafe live rclone configuration'
|
||||
chmod 600 "$rclone_path"
|
||||
fi
|
||||
install -m 600 -o root -g root "$UP_STATE/candidate.conf" "$CONFIG_FILE.update-tmp"
|
||||
mv -fT "$CONFIG_FILE.update-tmp" "$CONFIG_FILE"
|
||||
load_config
|
||||
build_plan
|
||||
for file in "${DB_CNFS[@]}"; do secure_file "$file"; chmod 600 "$file"; done
|
||||
# Hold the shared lock across every child. This also excludes manual restores.
|
||||
bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --check
|
||||
# Disable all retention in this acceptance run, including local retention.
|
||||
bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --no-retention
|
||||
backup_run=$(python3 - "$WORKDIR/last-backup.json" <<'PY'
|
||||
import json,sys
|
||||
value=json.load(open(sys.argv[1]))
|
||||
assert value['exit_code'] == 0
|
||||
print(value['run'])
|
||||
PY
|
||||
)
|
||||
bash /usr/local/sbin/app-restore.sh --config "$CONFIG_FILE" --remote-run "$backup_run" --verify-only
|
||||
printf '%s\n' "$backup_run" > "$UP_STATE/verified-backup-run"
|
||||
# A missed persistent timer can fire immediately, so release the shared lock first.
|
||||
unset APP_BACKUP_INHERIT_LOCK
|
||||
flock -u 9
|
||||
systemctl enable --now app-backup.timer
|
||||
UP_OK=true
|
||||
Reference in new issue
Block a user