diff --git a/.gitignore b/.gitignore index f773b59..ea9674e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ rclone.conf *.tar.zst __pycache__/ *.pyc + +# Built transfer packages are published separately from source control. +dist/ diff --git a/Dokumentation.md b/Dokumentation.md index 80d7e09..722e6b0 100644 --- a/Dokumentation.md +++ b/Dokumentation.md @@ -236,6 +236,34 @@ v1-Split-Archive haben weder manifestgebundene Prüfsummen noch einen verlässli Nur die Dateien **eines** alten Laufs in einen eigenen Ordner kopieren. Alte DB-Dumps liegen gemeinsam unter `db/`, Gitea unter `files/gitea-data`, Mail-Homes ggf. gesammelt unter `files/mail-home`. Nach Prüfung manuell in vorbereitete Testziele kopieren und DB-Dumps mit einem passenden Client importieren. Alte monolithische Formate vor den Split-Archiven unterstützt dieser Entpackmodus nicht. Alte Sicherungen erst nach einer erfolgreichen neuen Offsite-Sicherung und echten Wiederherstellungsprobe freigeben. +## Automatisiertes Transferpaket + +Ergänzend zum manuellen Installer gibt es ein selbstentpackendes `.run`-Paket mit +integrierter Payload-Prüfsumme und separater SHA-256-Datei. Es wird aus einer +expliziten Dateiliste gebaut; echte Zugangsdaten, Git-Historie und Sicherungsdaten +sind ausgeschlossen. Anleitung: [UPDATE_ANLEITUNG.md](UPDATE_ANLEITUNG.md). + +`update.sh` erstellt einen privaten Migrationsordner und eine Kandidatenkonfiguration. +Der erste Check nutzt geschützte Kopien der Zugangsdaten. Installation und +Abnahmebackup laufen dann unter einer gemeinsamen, von Kindprozessen überprüften +Dateisperre. Der Starter übergibt den langen Ablauf an eine systemd-Unit; das +Ergebnis bleibt im Migrationsordner erhalten. Ein kombinierter Rollback stellt +Paketdateien und die vorherige Hauptkonfiguration wieder her. + +Der Abnahmelauf nutzt `app-backup.sh --no-retention`: keinerlei lokale oder entfernte +Löschung alter Sicherungssätze. Nach Upload/Rücklesevergleich prüft zusätzlich +`app-restore.sh --remote-run ... --verify-only` den erneut heruntergeladenen Satz. +Erst danach wird der vorhandene Backup-Timer aktiviert. Remote-Retention bleibt +nach dieser Migration bewusst aus, bis die fachliche Restore-Abnahme erfolgt ist. + +Direkt erkennbare Nextcloud-systemd-Writer werden übernommen. Bei klassischen +Cronjobs fragt das Paket nach einer vorübergehenden Pause des gesamten Cron-Dienstes; +dies kann andere fällige Cronjobs betreffen und erfolgt deshalb nicht stillschweigend. +Nach dem Writer-Stopp wartet die Sicherung auf noch laufende direkte Nextcloud- +`cron.php`-/`occ`-Prozesse (maximal `WRITER_DRAIN_TIMEOUT`, Default 300 Sekunden). +Nextcloud dokumentiert systemd-Timer als Alternative zu crontab: +[Nextcloud Hintergrundjobs](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html). + ## Installation, Update und Rückweg Die ausführbare Schrittfolge steht in [README: bestehende Installation aktualisieren](README.md#bestehende-installation-aktualisieren). @@ -265,7 +293,7 @@ Die Entfernung der drei DB-Dateien aus dem aktuellen Baum löscht deren Historie Reproduzierbar mit `tests/Dockerfile` und `tests/run.sh`. Der Container bekommt das Repository nur lesbar und benötigt während der Tests kein Netzwerk. Die MariaDB läuft ohne TCP-Port in einem temporären Verzeichnis. systemd/occ werden simuliert; alle eigentlichen Archive, Dateikopien, Dumps/Imports und Crypt-Operationen sind real. -Ergebnis des lokalen Abschlusslaufs am 10.09.2026: **22 Tests erfolgreich**, Shell-Syntax und ShellCheck ohne Befund. Testumgebung: Debian 13.6, MariaDB 11.8.6, rclone 1.60.1-DEV (Distributionspaket), ShellCheck 0.10.0. Diese Versionsangaben beschreiben die Testumgebung, keine Empfehlung für die Produktivversion. +Ergebnis des lokalen Abschlusslaufs am 10.09.2026: **30 Tests erfolgreich**, Shell-Syntax und ShellCheck ohne Befund. Testumgebung: Debian 13.6, MariaDB 11.8.6, rclone 1.60.1-DEV (Distributionspaket), ShellCheck 0.10.0. Diese Versionsangaben beschreiben die Testumgebung, keine Empfehlung für die Produktivversion. Geprüfte Fälle umfassen: @@ -279,5 +307,8 @@ Geprüfte Fälle umfassen: - Selektiver Restore mit Löschung und Schutz ausgeschlossener Nextcloud-Daten. - Lokale/Remote-Retention mit Mindestanzahl und Erhalt alter/unvollständiger Läufe. - Paketupdate mit Erhalt von Config/Zeitplan und ausführbarem Rollback. +- Selbstentpackendes Transferpaket mit erfolgreichem Komplettlauf und kombiniertem Konfigurations-/Paket-Rollback. +- Abbruch vor Änderungen bei Vorprüfungsfehlern, gestoppter Timer bei fehlgeschlagenem Abnahmebackup, Ablehnung beschädigter Pakete. +- Übergabe an einen vom Starter unabhängigen Prozess (systemd-run im Test simuliert), ausdrückliche Cron-Policy und Writer-Wartezeit/Timeout. **Serverabnahme steht separat aus:** echte Rocky-Linux-10-Umgebung, SELinux-/ACL-Verhalten auf den verwendeten Dateisystemen, installierte Nextcloud-/Gitea-Versionen, vollständige Writer-/Pfadliste, Berechtigungen des echten DB-Backupaccounts, tatsächlicher OneDrive-Upload großer Archive, Laufzeit/Speicherbedarf und fachliche Restore-Probe. Es wurde kein Produktivbackup ausgeführt und keine Serverinstallation verändert. diff --git a/README.md b/README.md index ddfbea9..d698c6c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,25 @@ Sichert WordPress, Nextcloud, native Gitea-Installationen mit MariaDB, Maildaten Die vollständige Analyse, Architektur, Konfiguration, Betriebsanleitung und verbleibenden Grenzen stehen in [Dokumentation.md](Dokumentation.md). +## Automatisches Update-Paket (empfohlen) + +Das neue Paket übernimmt Konfigurationsabgleich, Rechteprüfung, Installation, +vollständigen Backup-Probelauf und Remote-Restore-Prüfung. Du benötigst auf dem +Server keinen Git-Checkout. Übertrage die gelieferte `.run`-Datei und ihre Prüfsumme +per SCP und starte im Zielverzeichnis: + +```bash +sha256sum -c app-backup-update-VERSION.run.sha256 +sudo bash app-backup-update-VERSION.run +``` + +VERSION durch den tatsächlichen Dateinamen ersetzen. Der längere Lauf arbeitet +anschließend als systemd-Auftrag unabhängig von der SSH-Verbindung. Bei unklaren +Gitea-/Cron-/rclone-Einstellungen fragt der Starter gezielt nach. Die Bedienung und +der kombinierte Rückweg stehen in [UPDATE_ANLEITUNG.md](UPDATE_ANLEITUNG.md). + +Die folgende manuelle Anleitung bleibt als Alternative für individuelle Updates. + ## Wesentliche Änderungen - Funktionierender Restore mit Prüflauf, Entpackmodus, Komponentenauswahl und ausdrücklichem `--apply --yes`. diff --git a/UPDATE_ANLEITUNG.md b/UPDATE_ANLEITUNG.md new file mode 100644 index 0000000..0033d63 --- /dev/null +++ b/UPDATE_ANLEITUNG.md @@ -0,0 +1,132 @@ +# Automatisches Update-Paket + +Auf dem Arbeitsplatz die gelieferte `.run`-Datei und ihre `.sha256`-Datei per SCP +auf den Server übertragen. Auf dem Server im selben Verzeichnis prüfen/starten: + +```bash +sha256sum -c app-backup-update-VERSION.run.sha256 +sudo bash app-backup-update-VERSION.run +``` + +VERSION durch den tatsächlichen Dateinamen ersetzen. Ein Git-Checkout auf dem +Server ist nicht erforderlich. Der Installer benötigt die vorhandene Konfiguration +`/etc/app-backup/app-backup.conf` und die bisherigen aktiven Zugangsdaten. + +Der Start ist die Freigabe für das Update einschließlich eines vollständigen +Backups mit zeitweiliger Unterbrechung der konfigurierten Anwendungen. + +## Was automatisch passiert + +1. Paketinhalt per SHA-256 prüfen und geschützt unter `/var/tmp` entpacken. +2. Bestehende Einstellungen nach `/var/backups/app-backup-upgrade-XXXXXXXX` sichern. +3. Aktuelle Pfade und Einstellungen übernehmen; sichere Betriebsoptionen ergänzen. +4. Vorhandene Webdienste und direkt erkennbare Nextcloud-systemd-Writer zuordnen. +5. Crypt-Konfiguration und DB-Dateien prüfen; vorhandene Schlüssel/Passwörter erhalten. +6. Vorprüfung mit geschützten Kopien durchführen. Bei Fehlern bleibt das installierte + Paket samt Anwendungskonfiguration und Timer unverändert. +7. Einen von SSH unabhängigen systemd-Auftrag `app-backup-upgrade.service` starten. +8. Backup-Timer stoppen, Paket mit Rollback-Inventar installieren, Konfiguration + atomar übernehmen und Rechte der aktiven root-eigenen Zugangsdaten korrigieren. +9. Vollständiges Backup ohne lokale/Remote-Retention durchführen. Anschließend den + abgeschlossenen Satz nochmals über Crypt herunterladen und mit dem Restore prüfen. +10. Erst bei Erfolg den Backup-Timer aktivieren und das Ergebnis protokollieren. + +Es findet kein Import in produktive Anwendungen oder Datenbanken statt. Der +Remote-Prüflauf belegt Archivlesbarkeit, ersetzt aber keinen fachlichen Test-Restore. + +## Verlauf und Ergebnis + +```bash +sudo journalctl -fu app-backup-upgrade.service +``` + +Der Starter nennt außerdem den **konkreten** Ergebnisordner: + +```bash +sudo cat /var/backups/app-backup-upgrade-XXXXXXXX/result.txt +sudo less /var/backups/app-backup-upgrade-XXXXXXXX/update.log +``` + +Erwartet wird `SUCCESS`. Solange `RUNNING` steht, nicht parallel erneut starten. +Nach Ende verschwindet die kurzlebige systemd-Unit; Log und Ergebnis bleiben erhalten. +Bei `QUEUED` wurde der Auftrag übergeben und wartet auf den Start. +`--foreground` führt den Vorgang stattdessen im aktuellen Terminal aus und ist +für Tests/Diagnose gedacht; dabei besteht keine Absicherung gegen SSH-Abbruch. + +## Gezielte Rückfragen und Grenzen + +- **Rclone-Datei nicht eindeutig:** Pfad im Dialog oder mit `--rclone-config FILE` + angeben. Eine user-eigene Datei wird in eine neue root-eigene Datei kopiert; + die vom Benutzer verwendete Originaldatei bleibt unverändert. +- **Gitea bisher nicht konfiguriert:** Aufnahme bestätigen oder bewusst + `--no-gitea` setzen. Absolute explizite Pfade und der MySQL-DB-Name können aus + `app.ini` übernommen werden. Mehrdeutige/relative Pfade müssen zuerst in der + vorhandenen Backup-Konfiguration ergänzt werden; es werden keine geratenen + Repository-Pfade eingetragen. +- **Nextcloud in crontab:** Der Installer fragt, ob der gefundene Cron-Dienst + während jeder Sicherung angehalten werden darf. Das betrifft auch andere + Cronjobs; währenddessen fällige Jobs können ausfallen. Alternativ Nextcloud + zuvor in einen eigenen systemd-Service/Timer übertragen. Bereits gestartete + direkte `cron.php`-/`occ`-Prozesse werden nach dem Writer-Stopp abgewartet + (Default 300 Sekunden), nicht blind beendet. Shell-Wrapper, externe Scheduler + und nicht direkt erkennbare Writer müssen explizit berücksichtigt werden. +- **Fehlende Basispakete:** Installation über `dnf` nach Rückfrage oder mit + `--install-deps`. Keine automatische Aktualisierung von PHP, DB-Server oder + rclone auf eine möglicherweise inkompatible Version. +- **Verschlüsselte rclone-Konfigurationsdatei:** Die für rclone benötigte + Entsperrung muss auch im systemd-Kontext verfügbar sein. Ohne verfügbare + Entsperrung bricht die Prüfung ab, statt unsichtbar auf eine Passworteingabe zu warten. Der Installer legt + keine neuen Crypt-Passwörter an und exportiert keine fremden Secret-Stores. +- **Früher eingecheckte DB-Passwörter:** Passwortrotation bleibt separat, weil + weitere Benutzer derselben Accounts nicht zuverlässig ermittelt werden können. + +Optionen können kombiniert werden: + +```bash +sudo bash app-backup-update-VERSION.run \ + --rclone-config /root/.config/rclone/rclone.conf \ + --enable-gitea --cron-services nextcloudcron.timer,nextcloudcron.service +``` + +Die Remote-Retention bleibt nach der Migration ausgeschaltet. Nach einem echten +Test-Restore bei Bedarf bewusst wieder aktivieren. Bestehende Timer-Zeitpläne +bleiben erhalten; beim Aktivieren kann `Persistent=true` einen Lauf nachholen. + +## Fehler und Rückweg + +Bei einem Fehler nach Beginn der Installation bleibt der Backup-Timer gestoppt. +Es erfolgt kein automatisches Zurücksetzen über möglicherweise noch laufende oder +teilweise wiederhergestellte Anwendungen. Logs und gegebenenfalls +`recovery-*.json` im Workdir prüfen. Das kombinierte Rollback stellt die vorherigen +Paketdateien **und** die vorherige Backup-Hauptkonfiguration wieder her: + +```bash +sudo bash /var/backups/app-backup-upgrade-XXXXXXXX/rollback-update.sh +``` + +Es verändert keine gesicherten Anwendungsdaten, rotiert keine Zugangsdaten und +startet den Timer nicht ungeprüft. Neu erzeugte root-eigene Kopien von rclone-Dateien +sowie strengere Dateirechte bleiben erhalten. Bei unterbrochener Vorbereitung +ohne Installationsbeginn ist kein Paket-Rollback erforderlich. + +## Paket reproduzieren + +Aus einem sauberen, eingecheckten Repository: + +```bash +python3 tools/build-update-package.py +``` + +Ausgabe: `dist/app-backup-update-COMMIT.run` und zugehörige SHA-256-Datei. +Das Paket enthält nur explizit freigegebene Programm-/Dokumentationsdateien und +Credential-Vorlagen; keine Git-Historie, echten CNF-Dateien oder Backupdaten. +Die Prüfsumme schützt vor Übertragungsfehlern, ist keine digitale Signatur. +Das ausführbare Paket ausschließlich über euren vertrauenswürdigen Kanal beziehen. + +## Verifikation der Lieferung + +30 Container-Tests sowie ShellCheck und Shell-Syntaxprüfung erfolgreich. +Die Tests enthalten echte MariaDB-Dumps/Imports und lokale rclone-Crypt-Roundtrips; +systemd/occ und die systemd-run-Übergabe sind simuliert. Die tatsächliche +Server-/OneDrive-Prüfung erfolgt beim ersten Paketlauf. Ein fachlicher Restore +auf einem separaten Testsystem bleibt erforderlich. diff --git a/install.sh b/install.sh index d1c84bd..c81d20a 100755 --- a/install.sh +++ b/install.sh @@ -5,8 +5,17 @@ umask 077 ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) [[ $EUID == 0 ]] || { echo 'Use sudo ./install.sh [--enable-timer]' >&2; exit 1; } ENABLE_TIMER=false -case ${1:-} in --enable-timer) ENABLE_TIMER=true ;; '') ;; *) echo 'Usage: sudo ./install.sh [--enable-timer]' >&2; exit 1 ;; esac -(( $# <= 1 )) || exit 1 +NO_TIMER=false +SNAPSHOT_DIR='' +while (( $# )); do + case $1 in + --enable-timer) ENABLE_TIMER=true; shift ;; + --no-timer) NO_TIMER=true; shift ;; + --snapshot-dir) SNAPSHOT_DIR=${2:?Missing snapshot directory}; shift 2 ;; + *) echo 'Usage: sudo ./install.sh [--enable-timer | --no-timer] [--snapshot-dir DIR]' >&2; exit 1 ;; + esac +done +[[ $NO_TIMER != true || $ENABLE_TIMER != true ]] || exit 1 # Syntax and packaging errors must be caught BEFORE changing the running installation. for file in "$ROOT_DIR"/usr-local-sbin/*.sh "$ROOT_DIR/lib/common.sh"; do bash -n "$file"; done python3 - "$ROOT_DIR/lib/backup_set.py" <<'PY' @@ -21,22 +30,27 @@ if [[ -f /etc/app-backup/app-backup.conf ]]; then # shellcheck disable=SC1091 source /etc/app-backup/app-backup.conf fi -[[ ! -L $LOCKFILE ]] || exit 1 -exec 9>"$LOCKFILE" -flock -n 9 || { echo 'Backup/restore is running. Retry after it finishes.' >&2; exit 1; } +# shellcheck source=lib/locking.sh +source "$ROOT_DIR/lib/locking.sh" +acquire_backup_lock if systemctl is-active --quiet app-backup.service; then echo 'app-backup.service is active. Wait for completion before updating.' >&2 exit 1 fi TIMER_ACTIVE=false if systemctl is-active --quiet app-backup.timer; then TIMER_ACTIVE=true; fi -SNAPSHOT=$(mktemp -d /var/backups/app-backup-update-XXXXXXXX) +if [[ -n $SNAPSHOT_DIR ]]; then + [[ $SNAPSHOT_DIR == /* && ! -e $SNAPSHOT_DIR ]] || { echo 'Snapshot directory must be new and absolute'; exit 1; } + mkdir -m 700 -- "$SNAPSHOT_DIR" + SNAPSHOT=$SNAPSHOT_DIR +else SNAPSHOT=$(mktemp -d /var/backups/app-backup-update-XXXXXXXX); fi chmod 700 "$SNAPSHOT" declare -a SOURCES TARGETS MODES add() { SOURCES+=("$ROOT_DIR/$1"); TARGETS+=("$2"); MODES+=("$3"); } add usr-local-sbin/app-backup.sh /usr/local/sbin/app-backup.sh 750 add usr-local-sbin/app-restore.sh /usr/local/sbin/app-restore.sh 750 add lib/common.sh /usr/local/lib/app-backup/common.sh 640 +add lib/locking.sh /usr/local/lib/app-backup/locking.sh 640 add lib/backup_set.py /usr/local/lib/app-backup/backup_set.py 640 add systemd/app-backup.service /etc/systemd/system/app-backup.service 644 # Preserve existing schedules. The new timer template remains available as .dist. @@ -50,6 +64,7 @@ else add etc/app-backup/app-backup.conf /etc/app-backup/app-backup.conf 600; fi for f in "$ROOT_DIR"/etc/app-backup/*.example; do add "etc/app-backup/$(basename "$f")" "/etc/app-backup/$(basename "$f")" 600; done add README.md /usr/local/share/doc/app-backup/README.md 644 add Dokumentation.md /usr/local/share/doc/app-backup/Dokumentation.md 644 +add UPDATE_ANLEITUNG.md /usr/local/share/doc/app-backup/UPDATE_ANLEITUNG.md 644 # Save the exact replaced files plus a rollback inventory; do not dereference links. : > "$SNAPSHOT/present.txt"; : > "$SNAPSHOT/absent.txt" for i in "${!TARGETS[@]}"; do @@ -69,8 +84,9 @@ set -euo pipefail here=$(cd -- "$(dirname -- "$0")" && pwd) LOCKFILE=/run/app-backup.lock if [[ -f /etc/app-backup/app-backup.conf ]]; then source /etc/app-backup/app-backup.conf; fi -exec 9>"$LOCKFILE" -flock -n 9 || { echo 'Backup/restore/update is running'; exit 1; } +# shellcheck disable=SC1091 +source "$here/locking.sh" +acquire_backup_lock if systemctl is-active --quiet app-backup.service; then echo 'Backup service is active'; exit 1; fi systemctl stop app-backup.timer while IFS= read -r target; do cp -a --remove-destination -- "$here$target" "$target"; done < "$here/present.txt" @@ -78,6 +94,8 @@ while IFS= read -r target; do rm -f -- "$target"; done < "$here/absent.txt" systemctl daemon-reload echo 'Previous package restored; config and timer state require manual review. Timer remains stopped.' ROLLBACK +cp -- "$ROOT_DIR/lib/locking.sh" "$SNAPSHOT/locking.sh" +chmod 600 "$SNAPSHOT/locking.sh" chmod 700 "$SNAPSHOT/rollback.sh" CHANGING=false on_exit() { @@ -104,8 +122,9 @@ install -d -m 700 /var/log/app-backup /var/backups/app-backup systemctl daemon-reload # A persistent timer can fire immediately. Release the package lock only after # all files are installed, but before that timer can start the new backup. -flock -u 9 -if [[ $ENABLE_TIMER == true ]]; then systemctl enable --now app-backup.timer +if [[ ${APP_BACKUP_INHERIT_LOCK:-0} != 1 ]]; then flock -u 9; fi +if [[ $NO_TIMER == true ]]; then : +elif [[ $ENABLE_TIMER == true ]]; then systemctl enable --now app-backup.timer elif [[ $TIMER_ACTIVE == true ]]; then systemctl start app-backup.timer; fi CHANGING=false echo "Installed. Rollback package: $SNAPSHOT" diff --git a/lib/backup_set.py b/lib/backup_set.py index c0867bf..dc439a4 100644 --- a/lib/backup_set.py +++ b/lib/backup_set.py @@ -13,6 +13,7 @@ from pathlib import Path, PurePosixPath import re import subprocess import sys +import time import tarfile FORMAT = 2 @@ -201,9 +202,38 @@ def write_status(folder, operation, run_id, code): os.replace(str(path) + ".tmp", path) +def wait_nextcloud(directory, timeout): + timeout = int(timeout) + if timeout < 1 or timeout > 3600: + raise ValueError("WRITER_DRAIN_TIMEOUT must be 1..3600 seconds") + expected = {os.path.realpath(directory + "/cron.php"), os.path.realpath(directory + "/occ")} + deadline = time.monotonic() + timeout + while True: + found = [] + for process in Path("/proc").iterdir(): + if not process.name.isdigit() or int(process.name) == os.getpid(): + continue + try: + args = (process / "cmdline").read_bytes().split(b"\0") + cwd = os.readlink(process / "cwd") + # Exact script arguments only: do not match this helper's NC_DIR + # argument, unrelated files, or arbitrary shell command strings. + if any(arg and os.path.realpath(os.path.join(cwd, os.fsdecode(arg))) in expected for arg in args): + found.append(process.name) + except (FileNotFoundError, ProcessLookupError): + continue + if not found: + return + if time.monotonic() >= deadline: + raise ValueError("Nextcloud writers still active (PIDs: " + ", ".join(found) + ")") + time.sleep(1) + + def main(): op, *args = sys.argv[1:] - if op == "status": + if op == "wait-nextcloud": + wait_nextcloud(*args) + elif op == "status": write_status(*args) elif op == "crypt": crypt(*args) diff --git a/lib/common.sh b/lib/common.sh index 5488800..6ea96b9 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -98,13 +98,9 @@ guard_workspaces() { init_runtime() { need python3; need flock guard_workspaces - # Both backup and restore, and the installer, take the same lock. - [[ $LOCKFILE == /* && ! -L $LOCKFILE ]] || die "Unsafe lock path" - if [[ -e $LOCKFILE ]]; then - [[ -f $LOCKFILE && ! -s $LOCKFILE && $(stat -c %u "$LOCKFILE") == 0 ]] || die "Unsafe existing lock file" - fi - exec 9>"$LOCKFILE" - flock -n 9 || die "Another backup/restore/update holds $LOCKFILE" + # shellcheck source=lib/locking.sh + source "$LIB_DIR/locking.sh" + acquire_backup_lock for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do [[ $p == /* && $p != / && $p != *$'\n'* && $p != *$'\t'* ]] || die "Unsafe working path: $p" [[ ! -L $p ]] || die "Symlink working directory refused: $p" @@ -221,8 +217,8 @@ check_remote() { done # Pipe directly into the parser: compatibility with older rclone without # config redacted. Secret values never go to the log or a temporary file. - "$RCLONE_BIN" --config "$RCLONE_CONFIG" config show "$name" | helper crypt "$name" - RCLONE_ARGS=(--config "$RCLONE_CONFIG" --no-check-certificate=false --crypt-filename-encryption standard + "$RCLONE_BIN" --ask-password=false --config "$RCLONE_CONFIG" config show "$name" | helper crypt "$name" + RCLONE_ARGS=(--ask-password=false --config "$RCLONE_CONFIG" --no-check-certificate=false --crypt-filename-encryption standard --crypt-directory-name-encryption=true --crypt-no-data-encryption=false --retries "$RCLONE_RETRIES" --low-level-retries "$RCLONE_LOW_LEVEL_RETRIES" --retries-sleep "$RCLONE_RETRIES_SLEEP" --timeout "$RCLONE_TIMEOUT" --contimeout "$RCLONE_CONTIMEOUT" @@ -289,6 +285,11 @@ quiesce() { if [[ $MAIL_BACKUP_MODE == stop ]]; then stop_service postfix; stop_service dovecot else echo "WARNING: live mail copy is not a consistent snapshot"; fi fi + if [[ $ENABLE_NEXTCLOUD == true ]]; then + # Cron daemons may leave an already launched PHP child running after stop. + # Wait for direct cron.php / occ processes to finish; never kill them blindly. + helper wait-nextcloud "$NC_DIR" "${WRITER_DRAIN_TIMEOUT:=300}" + fi } # Survives SIGKILL/power loss, which Bash traps cannot handle. This is an # instruction record for the operator, not an automatically executed recovery. diff --git a/lib/locking.sh b/lib/locking.sh new file mode 100644 index 0000000..1060c90 --- /dev/null +++ b/lib/locking.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# The upgrade coordinator holds FD 9 for the complete install/check/backup cycle. +# Child tools may reuse that SAME open file description, never bypass the lock. +acquire_backup_lock() { + [[ $LOCKFILE == /* && ! -L $LOCKFILE ]] || { echo 'Unsafe lock path' >&2; return 1; } + if [[ -e $LOCKFILE ]]; then + [[ -f $LOCKFILE && ! -s $LOCKFILE && $(stat -c %u "$LOCKFILE") == 0 ]] || { echo 'Unsafe existing lock file' >&2; return 1; } + fi + if [[ ${APP_BACKUP_INHERIT_LOCK:-0} == 1 ]]; then + [[ -e /proc/self/fd/9 && $(stat -Lc '%d:%i' /proc/self/fd/9) == "$(stat -Lc '%d:%i' "$LOCKFILE")" ]] || { + echo 'Inherited lock descriptor does not match configured lock' >&2; return 1; + } + else + exec 9>"$LOCKFILE" + fi + flock -n 9 || { echo "Another backup/restore/update holds $LOCKFILE" >&2; return 1; } +} diff --git a/lib/update-discover.py b/lib/update-discover.py new file mode 100644 index 0000000..a7e6ad0 --- /dev/null +++ b/lib/update-discover.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Read server metadata for migration without exposing config values or secrets.""" +import configparser +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys + + +def cron_units(nc_dir): + """Discover directly configured occ/cron.php writers, not arbitrary wrappers.""" + proc = subprocess.run(['systemctl','list-unit-files','--type=service','--no-legend','--no-pager'], check=True, text=True, capture_output=True) + services = [] + for line in proc.stdout.splitlines(): + unit = line.split()[0] + # Templates cannot be stopped without a concrete instance name. + if '@.' in unit: + continue + command = subprocess.check_output(['systemctl','show',unit,'--property=ExecStart','--value'],text=True) + if nc_dir in command and ('cron.php' in command or '/occ' in command): + services.append(unit) + timers = [] + for service in services: + triggered = subprocess.check_output(['systemctl','show',service,'--property=TriggeredBy','--value'],text=True) + timers.extend(x for x in triggered.split() if x.endswith(('.timer','.socket'))) + for unit in dict.fromkeys(timers + services): + if not re.fullmatch(r'[A-Za-z0-9_.@:-]+\.(service|timer|socket)', unit): + raise ValueError('Unexpected systemd unit name') + print(unit) + + +def check_crontabs(nc_dir, nc_user): + """Refuse to guess scheduling semantics for cron scripts; report locations only.""" + found = [] + paths = [Path('/etc/crontab')] + for directory in ('/etc/cron.d','/etc/cron.hourly','/etc/cron.daily','/etc/cron.weekly','/etc/cron.monthly','/var/spool/cron'): + root = Path(directory) + if root.is_dir(): + paths.extend(p for p in root.rglob('*') if p.is_file()) + for path in paths: + if not path.is_file(): + continue + for line in path.read_text(errors='replace').splitlines(): + if not line.lstrip().startswith('#') and (nc_dir in line or 'cron.php' in line): + found.append(str(path)) + break + if found: + print('Nextcloud cron entries: '+', '.join(sorted(set(found)))) + sys.exit(2) + + +def gitea(path): + parser = configparser.ConfigParser(interpolation=None, strict=False) + parser.read_string('[DEFAULT]\n' + Path(path).read_text()) + result = {} + # Only explicitly configured absolute filesystem paths are accepted here. + for key, section, option in [('GITEA_DATA_DIR','DEFAULT','APP_DATA_PATH'),('GITEA_REPOS_DIR','repository','ROOT')]: + value = parser[section].get(option,'') if section in parser else '' + if value and os.path.isabs(value) and '%' not in value: + result[key] = value + if parser.get('database','DB_TYPE',fallback='') == 'mysql': + result['GITEA_DB_NAME'] = parser.get('database','NAME',fallback='') + for key,value in result.items(): + print(key+'='+shlex.quote(value)) + + +def environment_config(): + raw = subprocess.check_output(['systemctl','show','app-backup.service','--property=Environment','--value'],text=True) + for item in shlex.split(raw): + if item.startswith('RCLONE_CONFIG='): + print(item.split('=',1)[1]) + return + + +if __name__ == '__main__': + try: + op,*args=sys.argv[1:] + if op == 'cron-units': cron_units(*args) + elif op == 'check-crontabs': check_crontabs(*args) + elif op == 'gitea': gitea(*args) + elif op == 'rclone-config': environment_config() + else: raise ValueError('Unknown discovery operation') + except (ValueError, OSError, configparser.Error, subprocess.CalledProcessError): + # Parser errors can include secret config lines, so do not echo exceptions. + print('ERROR: Discovery incomplete. Check Nextcloud cron scheduling or explicit Gitea/rclone settings; no secret values logged.',file=sys.stderr) + sys.exit(1) diff --git a/tests/run.sh b/tests/run.sh index 00e4f82..4ac702d 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -Eeuo pipefail cd "$(dirname "$0")/.." -for script in lib/common.sh usr-local-sbin/*.sh install.sh tests/run.sh; do bash -n "$script"; done -shellcheck -x lib/common.sh usr-local-sbin/*.sh install.sh tests/run.sh +for script in lib/*.sh usr-local-sbin/*.sh install.sh update.sh tests/run.sh; do bash -n "$script"; done +shellcheck -x lib/*.sh usr-local-sbin/*.sh install.sh update.sh tests/run.sh python3 -m unittest discover -s tests -v diff --git a/tests/test_backup.py b/tests/test_backup.py index 63d659a..8bda708 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -55,6 +55,23 @@ class ArchiveValidation(unittest.TestCase): with self.assertRaises((ValueError, tarfile.TarError)): HELPER.check_tar(path, ['files/wordpress']) + def test_writer_drain_waits_and_reports_timeout(self): + with tempfile.TemporaryDirectory() as directory: + writer=subprocess.Popen(['python3','-c','import time; time.sleep(1)',directory+'/cron.php']) + try: + started=time.monotonic() + HELPER.wait_nextcloud(directory,3) + self.assertGreaterEqual(time.monotonic()-started,.5) + finally: + writer.wait(timeout=5) + writer=subprocess.Popen(['python3','-c','import time; time.sleep(5)',directory+'/occ']) + try: + with self.assertRaisesRegex(ValueError,'writers still active'): + HELPER.wait_nextcloud(directory,1) + finally: + writer.terminate() + writer.wait(timeout=5) + def test_retention_keeps_minimum_and_ignores_v1(self): result = subprocess.run(['python3', str(REPO/'lib/backup_set.py'), 'candidates', 'appbackup', '7', '2'], input='appbackup_2020-01-01_00-00-00_aaaaaaaa\nappbackup_2020-01-02_00-00-00_bbbbbbbb\nappbackup_2020-01-03_00-00-00_cccccccc\nappbackup_2019-01-01_00-00-00\n../evil\n', @@ -119,12 +136,16 @@ with open(os.environ['TEST_EVENTS'],'a') as f: f.write(name+' '+' '.join(args)+' if name=='systemctl': cmd=args[0] service=args[1] if len(args)>1 else '' - if cmd=='show': print(s['services'].get(service,'inactive')) + if cmd=='show': + if '--property=LoadState' in args: print('loaded' if service in s['services'] else 'not-found') + elif '--property=Environment' in args or '--property=TriggeredBy' in args: print('') + else: print(s['services'].get(service,'inactive')) elif cmd=='is-active': sys.exit(0 if s['services'].get(args[-1])=='active' else 3) elif cmd in ('stop','start'): if os.environ.get('TEST_FAIL_SERVICE')==cmd+':'+service: sys.exit(1) s['services'][service]='active' if cmd=='start' else 'inactive' - elif cmd in ('daemon-reload','enable'): pass + elif cmd=='enable': s['services'][args[-1]]='active' + elif cmd=='daemon-reload': pass elif name=='sudo': os.execvp(args[2],args[2:]) elif name=='php': @@ -373,6 +394,130 @@ QUIESCE_SERVICES=(nc-cron) self.run_script('app-backup.sh','--check',ok=False) self.assertEqual(self.wp.stat().st_mode & 0o777,0o755) + def build_package(self): + output=self.base/'package' + subprocess.run(['python3',str(REPO/'tools/build-update-package.py'),'--version','integration','--output',str(output)],check=True,capture_output=True) + return output/'app-backup-update-integration.run' + + def configure_upgrade(self): + self.crypt_config() + canonical=Path('/etc/app-backup/app-backup.conf') + canonical.parent.mkdir(exist_ok=True) + shutil.copyfile(self.config,canonical) + canonical.chmod(0o600) + # Old installations commonly left credential files group-readable. + self.cnf.chmod(0o640) + state=json.loads(self.state.read_text()) + state['services']['app-backup.timer']='active' + self.state.write_text(json.dumps(state)) + return canonical + + def test_automatic_package_upgrade_and_combined_rollback(self): + canonical=self.configure_upgrade() + original=canonical.read_bytes() + package=self.build_package() + result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=120) + self.assertEqual(result.returncode,0,result.stdout+result.stderr) + state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0]) + self.assertIn('SUCCESS',(state/'result.txt').read_text()) + self.assertTrue((state/'verified-backup-run').is_file()) + self.assertIn('ENABLE_REMOTE_RETENTION=false',canonical.read_text()) + self.assertEqual(self.cnf.stat().st_mode & 0o777,0o600) + self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active') + rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30) + self.assertEqual(rollback.returncode,0,rollback.stdout+rollback.stderr) + self.assertEqual(canonical.read_bytes(),original) + self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'inactive') + + def test_package_preflight_failure_preserves_live_configuration_and_timer(self): + canonical=self.configure_upgrade() + original=canonical.read_bytes() + package=self.build_package() + result=subprocess.run(['bash',str(package),'--foreground'],env=dict(self.env,TEST_FAIL_TOOL='mariadb'),capture_output=True,text=True,timeout=60) + self.assertNotEqual(result.returncode,0,result.stdout+result.stderr) + self.assertEqual(canonical.read_bytes(),original) + self.assertEqual(self.cnf.stat().st_mode & 0o777,0o640) + self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active') + self.assertNotIn('systemctl stop',self.events.read_text()) + + def test_package_failed_backup_keeps_timer_stopped_and_rollback_available(self): + canonical=self.configure_upgrade() + original=canonical.read_bytes() + package=self.build_package() + result=subprocess.run(['bash',str(package),'--foreground'],env=dict(self.env,TEST_FAIL_TOOL='mariadb-dump'),capture_output=True,text=True,timeout=120) + self.assertNotEqual(result.returncode,0,result.stdout+result.stderr) + state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0]) + self.assertIn('FAILED',(state/'result.txt').read_text()) + self.assertTrue((state/'rollback-update.sh').is_file()) + self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'inactive') + self.assertFalse((state/'verified-backup-run').exists()) + rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30) + self.assertEqual(rollback.returncode,0,rollback.stdout+rollback.stderr) + self.assertEqual(canonical.read_bytes(),original) + + def test_package_corruption_is_rejected_before_preparation(self): + package=self.build_package() + original=package.read_bytes() + package.write_bytes(original[:-1]+bytes([original[-1]^1])) + result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=20) + self.assertNotEqual(result.returncode,0) + self.assertIn('Paket-Pruefsumme falsch',result.stderr) + self.assertNotIn('systemctl',self.events.read_text()) + + def test_inherited_lock_flag_cannot_bypass_another_process(self): + import fcntl + with (self.base/'lock').open('w') as lock: + fcntl.flock(lock,fcntl.LOCK_EX | fcntl.LOCK_NB) + result=self.run_script('app-backup.sh','--check',ok=False,APP_BACKUP_INHERIT_LOCK='1') + self.assertIn('Inherited lock descriptor',result.stdout) + + def test_detached_package_handoff_finishes_after_starter_exits(self): + self.configure_upgrade() + launcher=self.bin/'systemd-run' + launcher.write_text('''#!/usr/bin/env python3 +import os,subprocess,sys +from pathlib import Path +args=sys.argv[1:] +assert '--unit=app-backup-upgrade' in args +command=args[args.index('/bin/bash'):] +state=Path(command[-1]) +with open(state/'test-worker.log','w') as log: + subprocess.Popen(command,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,env=os.environ,close_fds=True,start_new_session=True) +''') + launcher.chmod(0o755) + package=self.build_package() + result=subprocess.run(['bash',str(package)],env=self.env,capture_output=True,text=True,timeout=60) + self.assertEqual(result.returncode,0,result.stdout+result.stderr) + state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0]) + for _ in range(120): + status=(state/'result.txt').read_text() if (state/'result.txt').exists() else '' + if status.startswith(('SUCCESS','FAILED')): break + time.sleep(.5) + self.assertTrue(status.startswith('SUCCESS'),status+(state/'test-worker.log').read_text()) + self.assertIn('unabhaengig von SSH',result.stdout) + + def test_cron_inventory_requires_explicit_policy_and_sorts_timer_first(self): + canonical=self.configure_upgrade() + cron=Path('/etc/cron.d/app-backup-test-nextcloud') + cron.parent.mkdir(exist_ok=True) + cron.write_text(f'*/5 * * * * root /usr/bin/php -f {self.nc}/cron.php\n') + self.addCleanup(cron.unlink) + state=json.loads(self.state.read_text()) + state['services']['crond.service']='active' + state['services']['nc-cron.timer']='active' + self.state.write_text(json.dumps(state)) + package=self.build_package() + result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=60) + self.assertNotEqual(result.returncode,0,result.stdout+result.stderr) + self.assertIn('noninteractive',result.stdout+result.stderr) + self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active') + result=subprocess.run(['bash',str(package),'--foreground','--cron-services','nc-cron,crond.service,nc-cron.timer'],env=self.env,capture_output=True,text=True,timeout=120) + self.assertEqual(result.returncode,0,result.stdout+result.stderr) + text=canonical.read_text() + line=[line for line in text.splitlines() if line.startswith('QUIESCE_SERVICES=')][-1] + self.assertLess(line.index('nc-cron.timer'),line.index('crond.service')) + self.assertEqual(json.loads(self.state.read_text())['services']['crond.service'],'active') + def crypt_config(self): (self.base/'encrypted').mkdir() password=subprocess.check_output(['/usr/bin/rclone','obscure','integration-test-only'],text=True).strip() diff --git a/tools/build-update-package.py b/tools/build-update-package.py new file mode 100644 index 0000000..6c595b7 --- /dev/null +++ b/tools/build-update-package.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Build a single self-extracting installer from an explicit, secret-free allowlist.""" +import argparse +import gzip +import hashlib +import io +from pathlib import Path +import re +import subprocess +import tarfile + +ROOT = Path(__file__).resolve().parents[1] +FILES = ['install.sh', 'update.sh', 'README.md', 'Dokumentation.md', 'UPDATE_ANLEITUNG.md', + 'lib/common.sh', 'lib/locking.sh', 'lib/backup_set.py', 'lib/update-discover.py', + 'usr-local-sbin/app-backup.sh', 'usr-local-sbin/app-restore.sh', + 'systemd/app-backup.service', 'systemd/app-backup.timer', 'logrotate/app-backup', + 'etc/app-backup/app-backup.conf'] + + +def build(output, version): + if not re.fullmatch(r'[A-Za-z0-9._-]+', version): + raise ValueError('Unsafe version identifier') + files = FILES + [str(p.relative_to(ROOT)) for p in sorted((ROOT/'etc/app-backup').glob('*.cnf.example'))] + buffer = io.BytesIO() + with gzip.GzipFile(fileobj=buffer, mode='wb', mtime=0) as zipped: + with tarfile.open(fileobj=zipped, mode='w') as archive: + for name in sorted(files): + path = ROOT / name + if path.is_symlink() or not path.is_file(): + raise ValueError(f'Missing/unsafe package input: {name}') + raw = path.read_bytes() + info = tarfile.TarInfo(name) + info.size = len(raw) + info.mode = 0o700 if name.endswith('.sh') else 0o600 + archive.addfile(info, io.BytesIO(raw)) + raw = (version + '\n').encode() + info = tarfile.TarInfo('VERSION') + info.size = len(raw) + info.mode = 0o600 + archive.addfile(info, io.BytesIO(raw)) + payload = buffer.getvalue() + checksum = hashlib.sha256(payload).hexdigest() + # Payload checksum catches transfer corruption. It is not a signature; obtain + # this executable only through the trusted SSH/source repository channel. + header = f'''#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +if [[ ${{1:-}} == --help ]]; then + echo 'Usage: sudo bash app-backup-update-{version}.run [--foreground] [--config FILE] [--rclone-config FILE] [--enable-gitea | --no-gitea] [--cron-services UNIT,...] [--install-deps]' + exit 0 +fi +[[ $EUID == 0 ]] || {{ echo 'Bitte mit sudo bash starten.' >&2; exit 1; }} +for tool in sha256sum tar mktemp tail awk; do command -v "$tool" >/dev/null; done +line=$(awk '/^__APP_BACKUP_PAYLOAD__$/ {{print NR + 1; exit}}' "$0") +[[ -n $line ]] || exit 1 +actual=$(tail -n +"$line" "$0" | sha256sum) +[[ ${{actual%% *}} == {checksum} ]] || {{ echo 'Paket-Pruefsumme falsch; keine Installation.' >&2; exit 1; }} +# Root-owned persistent extraction survives SSH disconnect and worker startup. +package_dir=$(mktemp -d /var/tmp/app-backup-package-{version}-XXXXXXXX) +tail -n +"$line" "$0" | tar -xz --no-same-owner -C "$package_dir" +exec bash "$package_dir/update.sh" "$@" +exit 1 +__APP_BACKUP_PAYLOAD__ +'''.encode() + output.mkdir(parents=True, exist_ok=True) + package = output / f'app-backup-update-{version}.run' + package.write_bytes(header + payload) + package.chmod(0o700) + (output / (package.name+'.sha256')).write_text(hashlib.sha256(package.read_bytes()).hexdigest()+' '+package.name+'\n') + return package + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--output', type=Path, default=ROOT/'dist') + parser.add_argument('--version', help='Override for development/testing only') + args = parser.parse_args() + version = args.version + if not version: + if subprocess.check_output(['git','status','--porcelain'],cwd=ROOT).strip(): + parser.error('Commit source changes before creating a release package') + version = subprocess.check_output(['git','rev-parse','--short=12','HEAD'],cwd=ROOT,text=True).strip() + print(build(args.output, version)) diff --git a/update.sh b/update.sh new file mode 100755 index 0000000..d61c7a2 --- /dev/null +++ b/update.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +# Upgrade coordinator: prepare locally, then run the long operation under systemd. +# No Git checkout, credential template replacement, or application restore needed. +set -Eeuo pipefail +umask 077 +UPDATE_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +LIB_DIR="$UPDATE_ROOT/lib" +# shellcheck source=lib/common.sh +source "$LIB_DIR/common.sh" +# shellcheck source=lib/locking.sh +source "$LIB_DIR/locking.sh" +CONFIG_FILE=/etc/app-backup/app-backup.conf +UP_FOREGROUND=false +UP_INSTALL_DEPS=false +UP_GITEA='' +UP_RCLONE='' +UP_CRON='' +UP_STATE='' +UP_WORKER=false +while (( $# )); do + case $1 in + --config) CONFIG_FILE=${2:?Missing configuration}; shift 2 ;; + --foreground) UP_FOREGROUND=true; shift ;; + --install-deps) UP_INSTALL_DEPS=true; shift ;; + --enable-gitea) UP_GITEA=true; shift ;; + --no-gitea) UP_GITEA=false; shift ;; + --rclone-config) UP_RCLONE=${2:?Missing rclone configuration}; shift 2 ;; + --cron-services) UP_CRON=${2:?Missing comma-separated systemd units}; shift 2 ;; + --worker) UP_WORKER=true; UP_STATE=${2:?Missing state}; shift 2 ;; + --help) echo 'Usage: sudo bash update.sh [--foreground] [--config FILE] [--rclone-config FILE] [--enable-gitea | --no-gitea] [--cron-services UNIT,...] [--install-deps]'; exit 0 ;; + *) die "Unknown argument: $1" ;; + esac +done +[[ $EUID == 0 ]] || die 'Run with sudo bash' +need systemctl; need flock +# This lock excludes two coordinators, independently from ordinary backup jobs. +exec 8>/run/app-backup-upgrade.lock +if [[ $UP_WORKER == true ]]; then flock -w 60 8; else flock -n 8 || die 'Another updater is preparing/running'; fi + +prompt() { + local answer + [[ -t 0 ]] || die "$1 (noninteractive: supply the matching command-line option/configuration)" + read -r -p "$1 " answer + [[ $answer == j || $answer == J || $answer == ja || $answer == y || $answer == yes ]] +} +append_setting() { printf '%s=%q\n' "$1" "$2" >> "$UP_STATE/candidate.conf"; } +discover() { python3 "$LIB_DIR/update-discover.py" "$@"; } + +if [[ $UP_WORKER == false ]]; then + systemctl is-active --quiet app-backup-upgrade.service && die 'Upgrade service is already running' + # Only install missing, small prerequisites when requested. Never upgrade PHP, + # database server or rclone to a guessed version as a side effect of this update. + missing=() + for pair in python3:python3 rsync:rsync zstd:zstd gzip:gzip tar:tar sudo:sudo; do + tool=${pair%%:*}; package=${pair#*:} + command -v "$tool" >/dev/null || missing+=("$package") + done + if (( ${#missing[@]} )); then + if [[ $UP_INSTALL_DEPS == true ]] || prompt "Fehlende Basispakete installieren: ${missing[*]}? [j/N]"; then + need dnf + dnf install -y "${missing[@]}" + else die "Missing prerequisites: ${missing[*]}"; fi + fi + [[ $CONFIG_FILE == /etc/app-backup/app-backup.conf ]] || die 'The service uses /etc/app-backup/app-backup.conf; migrate that canonical configuration' + secure_file "$CONFIG_FILE" + UP_STATE=$(mktemp -d /var/backups/app-backup-upgrade-XXXXXXXX) + chmod 700 "$UP_STATE" + cp -a -- "$CONFIG_FILE" "$UP_STATE/original.conf" + printf '%s\n' "$CONFIG_FILE" > "$UP_STATE/config-path" + sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1 > "$UP_STATE/original.sha256" + cp -a /etc/app-backup "$UP_STATE/etc-app-backup" + echo "Update-Sicherung und Protokoll: $UP_STATE" + # Read only trusted local Bash configuration; remote metadata is never sourced. + # shellcheck disable=SC1090 + source "$CONFIG_FILE" + UP_HAD_GITEA=${ENABLE_GITEA+x} + UP_ORIGINAL_RCLONE=${RCLONE_CONFIG:-} + load_config + acquire_backup_lock + # Preserve original text and append effective safety settings. Repeated upgrades + # may append another small block; the last explicit values remain authoritative. + cp -- "$CONFIG_FILE" "$UP_STATE/candidate.conf" + chmod 600 "$UP_STATE/candidate.conf" + printf '\n# Automated package migration (original preserved in update snapshot)\n' >> "$UP_STATE/candidate.conf" + append_setting ENABLE_REMOTE_RETENTION false + # Retention of existing local sets is also skipped for the acceptance backup. + append_setting ENABLE_UPLOAD true + append_setting ENABLE_NEXTCLOUD_MAINTENANCE true + append_setting MAIL_BACKUP_MODE stop + if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then + for pair in "ENABLE_HTTPD_STOP:$HTTPD_SERVICE_NAME" "ENABLE_PHPFPM_STOP:$PHPFPM_SERVICE_NAME"; do + name=${pair%%:*}; unit=${pair#*:} + state=$(systemctl show "$unit" --property=LoadState --value) + if [[ $state == loaded ]]; then append_setting "$name" true + elif [[ $state == not-found ]]; then append_setting "$name" false + else die "Cannot inspect web service: $unit ($state)"; fi + done + fi + if [[ -n $UP_GITEA ]]; then + ENABLE_GITEA=$UP_GITEA + elif [[ -z $UP_HAD_GITEA && $(systemctl show "$GITEA_SERVICE_NAME" --property=LoadState --value) == loaded ]]; then + if prompt 'Gitea wurde gefunden, bisher aber nicht explizit konfiguriert. In die Sicherung aufnehmen? [j/N]'; then ENABLE_GITEA=true + else die 'Choose --no-gitea explicitly if Gitea should remain excluded'; fi + fi + if [[ $ENABLE_GITEA == true && -z $UP_HAD_GITEA ]]; then + [[ -f $GITEA_ETC_DIR/app.ini ]] || die 'Gitea app.ini missing; configure Gitea paths first' + discover gitea "$GITEA_ETC_DIR/app.ini" > "$UP_STATE/gitea-paths.conf" + # Output contains ONLY allowlisted shell-quoted path/DB-name assignments. + # shellcheck disable=SC1090,SC1091 + source "$UP_STATE/gitea-paths.conf" + [[ -n ${GITEA_REPOS_DIR:-} && -n ${GITEA_DB_NAME:-} ]] || die 'Gitea repository path/database not explicit; set GITEA_REPOS_DIR and GITEA_DB_NAME in the existing config' + : "${GITEA_DB_CNF:=/etc/app-backup/db-gitea.cnf}" + for name in GITEA_DATA_DIR GITEA_ETC_DIR GITEA_REPOS_DIR GITEA_DB_NAME GITEA_DB_CNF; do append_setting "$name" "${!name}"; done + fi + append_setting ENABLE_GITEA "$ENABLE_GITEA" + append_setting ENABLE_GITEA_SERVICE_STOP true + [[ $ENABLE_GITEA == true ]] || echo 'Hinweis: Gitea bleibt entsprechend der Konfiguration deaktiviert.' + + # Keep keys/tokens. If a user-owned config was used before, copy it into a + # dedicated root-owned file; never chown a user's actively used configuration. + if [[ -n $UP_RCLONE ]]; then RCLONE_CONFIG=$UP_RCLONE + elif [[ -n $UP_ORIGINAL_RCLONE ]]; then RCLONE_CONFIG=$UP_ORIGINAL_RCLONE + else + detected=$(discover rclone-config) + if [[ -n $detected ]]; then RCLONE_CONFIG=$detected + else + RCLONE_CONFIG=$("$RCLONE_BIN" config file | tail -n 1) + fi + fi + if [[ ! -f $RCLONE_CONFIG || -L $RCLONE_CONFIG ]]; then + [[ -t 0 ]] || die 'rclone config not found; supply --rclone-config FILE' + read -r -p 'Pfad zur bisher verwendeten rclone.conf: ' RCLONE_CONFIG + fi + [[ -f $RCLONE_CONFIG && ! -L $RCLONE_CONFIG ]] || die 'rclone config is not a regular file' + cp -- "$RCLONE_CONFIG" "$UP_STATE/rclone.conf" + chmod 600 "$UP_STATE/rclone.conf" + # Use the original root-owned path where possible, preserving OAuth refreshes. + if [[ $(stat -c %u "$RCLONE_CONFIG") != 0 ]]; then + RCLONE_CONFIG=/etc/app-backup/rclone-migrated.conf + [[ ! -e $RCLONE_CONFIG ]] || die 'rclone-migrated.conf already exists; specify it explicitly' + fi + append_setting RCLONE_CONFIG "$RCLONE_CONFIG" + printf '%s\n' "$RCLONE_CONFIG" > "$UP_STATE/rclone-path" + + if [[ -n $UP_CRON ]]; then IFS=, read -r -a additional <<< "$UP_CRON"; QUIESCE_SERVICES+=("${additional[@]}"); fi + if [[ $ENABLE_NEXTCLOUD == true ]]; then + discover cron-units "$NC_DIR" > "$UP_STATE/cron-units" + mapfile -t detected_units < "$UP_STATE/cron-units" + QUIESCE_SERVICES+=("${detected_units[@]}") + # Arbitrary cron wrappers cannot be safely translated into systemd timers. + # Offer a precise policy choice rather than silently omit an active writer. + cron_result=0 + discover check-crontabs "$NC_DIR" "$NC_OCC_USER" || cron_result=$? + [[ $cron_result == 0 || $cron_result == 2 ]] || die 'Cron inventory failed; no safe writer plan available' + if [[ $cron_result == 2 ]]; then + cron_unit=crond.service + if [[ $(systemctl show "$cron_unit" --property=LoadState --value) != loaded ]]; then cron_unit=cron.service; fi + [[ $(systemctl show "$cron_unit" --property=LoadState --value) == loaded ]] || die 'Cannot identify cron service; configure writer shutdown explicitly' + if [[ " ${QUIESCE_SERVICES[*]} " != *" $cron_unit "* && " ${QUIESCE_SERVICES[*]} " != *" ${cron_unit%.service} "* ]]; then + prompt "Nextcloud-cron gefunden. $cron_unit waehrend jeder Sicherung pausieren (betrifft auch andere Cronjobs)? [j/N]" || die 'Move the Nextcloud job to a dedicated systemd service/timer, or explicitly supply --cron-services' + QUIESCE_SERVICES+=("$cron_unit") + fi + fi + fi + printf 'QUIESCE_SERVICES=(' >> "$UP_STATE/candidate.conf" + # De-duplicate: a timer must precede its service so resume can reverse the order. + for unit in "${QUIESCE_SERVICES[@]}"; do + [[ $unit =~ ^[a-zA-Z0-9_.@:-]+$ ]] || die "Invalid service name: $unit" + done + printf '%s\n' "${QUIESCE_SERVICES[@]}" | python3 -c 'import sys; units=list(dict.fromkeys(s.strip() for s in sys.stdin if s.strip())); print("\n".join(sorted(units, key=lambda s: not s.endswith((".timer", ".socket")))))' > "$UP_STATE/writers" + while IFS= read -r unit; do printf '%q ' "$unit" >> "$UP_STATE/candidate.conf"; done < "$UP_STATE/writers" + printf ')\n' >> "$UP_STATE/candidate.conf" + printf '%s\n' "$LOCKFILE" > "$UP_STATE/lock-path" + # --check uses protected copies of credentials, so preparation changes no live + # credential permissions and cannot break an older still-scheduled backup. + cp "$UP_STATE/candidate.conf" "$UP_STATE/check.conf" + printf '\nRCLONE_CONFIG=%q\n' "$UP_STATE/rclone.conf" >> "$UP_STATE/check.conf" + # Load the candidate to pick up newly enabled applications and their CNFs. + CONFIG_FILE="$UP_STATE/candidate.conf" + load_config + build_plan + for i in "${!DB_CNFS[@]}"; do + file=${DB_CNFS[i]} + secure_file "$file" + cp -- "$file" "$UP_STATE/db-$i.cnf" + chmod 600 "$UP_STATE/db-$i.cnf" + case ${DB_LABELS[i]} in db-wordpress) key=WP_DB_CNF ;; db-nextcloud) key=NC_DB_CNF ;; db-gitea) key=GITEA_DB_CNF ;; *) die 'Unknown database label' ;; esac + printf '%s=%q\n' "$key" "$UP_STATE/db-$i.cnf" >> "$UP_STATE/check.conf" + done + APP_BACKUP_INHERIT_LOCK=1 bash "$UPDATE_ROOT/usr-local-sbin/app-backup.sh" --config "$UP_STATE/check.conf" --check > "$UP_STATE/preflight.log" 2>&1 || { + tail -n 40 "$UP_STATE/preflight.log"; die "Preflight failed; installed package/config/timer unchanged. Details: $UP_STATE"; + } + sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1 > "$UP_STATE/candidate.sha256" + echo "Vorpruefung erfolgreich. Writer: $(tr '\n' ' ' < "$UP_STATE/writers")" + echo 'Jetzt folgen Installation, ein vollstaendiges Backup und Remote-Restore-Pruefung.' + if [[ $UP_FOREGROUND == true ]]; then + flock -u 9; flock -u 8 + exec bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" + fi + # A new systemd service inherits no shell FDs; it acquires both locks itself. + printf 'QUEUED\n' > "$UP_STATE/result.txt" + systemd-run --unit=app-backup-upgrade --collect --property=Type=oneshot \ + --property=TimeoutStartSec=24h --property=TimeoutStopSec=5min --property=KillMode=mixed \ + /bin/bash "$UPDATE_ROOT/update.sh" --worker "$UP_STATE" || { + printf 'FAILED: systemd worker could not be started; installed package/config/timer unchanged.\n' > "$UP_STATE/result.txt" + exit 1 + } + echo 'Update laeuft unabhaengig von SSH weiter.' + echo 'Verlauf: sudo journalctl -fu app-backup-upgrade.service' + echo "Ergebnis: sudo cat $UP_STATE/result.txt" + exit 0 +fi + +# -------------------- detached worker -------------------- +[[ $UP_STATE == /var/backups/app-backup-upgrade-* && -d $UP_STATE && ! -L $UP_STATE && $(stat -c %u "$UP_STATE") == 0 ]] || die 'Invalid upgrade state directory' +(( (8#$(stat -c %a "$UP_STATE") & 8#077) == 0 )) || die 'Upgrade state must be private' +exec > >(tee -a "$UP_STATE/update.log") 2>&1 +CONFIG_FILE=$(cat "$UP_STATE/config-path") +LOCKFILE=$(cat "$UP_STATE/lock-path") +UP_CHANGED=false +UP_OK=false +worker_finish() { + local code=$? + trap - EXIT INT TERM + set +e + if [[ $UP_OK == true && $code == 0 ]]; then + printf 'SUCCESS\nPaket aktualisiert, Backup und Remote-Restore-Pruefung erfolgreich.\nBackup-Timer aktiv.\n' > "$UP_STATE/result.txt" + else + if [[ $UP_CHANGED == true ]]; then systemctl stop app-backup.timer; fi + printf 'FAILED (exit %s)\nDetails: %s/update.log\n' "$code" "$UP_STATE" > "$UP_STATE/result.txt" + if [[ -f $UP_STATE/rollback-update.sh ]]; then + printf 'Rollback: sudo bash %s/rollback-update.sh\n' "$UP_STATE" >> "$UP_STATE/result.txt" + else echo 'Installation not started; package/config unchanged.' >> "$UP_STATE/result.txt"; fi + fi + cat "$UP_STATE/result.txt" + exit "$code" +} +trap worker_finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +printf 'RUNNING\n' > "$UP_STATE/result.txt" +acquire_backup_lock +export APP_BACKUP_INHERIT_LOCK=1 +[[ $(sha256sum "$CONFIG_FILE" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/original.sha256")" ]] || die 'Configuration changed after preparation; rerun updater' +[[ $(sha256sum "$UP_STATE/candidate.conf" | cut -d ' ' -f 1) == "$(cat "$UP_STATE/candidate.sha256")" ]] || die 'Prepared configuration changed unexpectedly' +systemctl stop app-backup.timer +UP_CHANGED=true +# The ordinary installer records all replaced package files. Its timer remains +# disabled here until this coordinator has verified the complete new backup. +cp -- "$LIB_DIR/locking.sh" "$UP_STATE/locking.sh" +cat > "$UP_STATE/rollback-update.sh" <<'ROLLBACK' +#!/usr/bin/env bash +set -Eeuo pipefail +[[ $EUID == 0 ]] || exit 1 +here=$(cd -- "$(dirname -- "$0")" && pwd) +config=$(cat "$here/config-path") +lock=$(cat "$here/lock-path") +# Package rollback acquires this same lock itself. Keep a separate coordinator +# lock across package rollback + config restoration to exclude another updater. +exec 8>/run/app-backup-upgrade.lock +flock -n 8 || { echo 'Update still active'; exit 1; } +systemctl stop app-backup.timer +LOCKFILE=$lock +source "$here/locking.sh" +acquire_backup_lock +export APP_BACKUP_INHERIT_LOCK=1 +if [[ -f $here/package-rollback/rollback.sh ]]; then + bash "$here/package-rollback/rollback.sh" +else echo 'No package rollback inventory exists; inspect installer log before proceeding.'; exit 1; fi +install -m 600 -o root -g root "$here/original.conf" "$config.rollback-tmp" +mv -fT "$config.rollback-tmp" "$config" +echo 'Package and previous application-backup configuration restored. Timer remains stopped.' +echo 'Credentials/keys were not rotated. Review services and data before restarting schedules.' +ROLLBACK +chmod 700 "$UP_STATE/rollback-update.sh" +bash "$UPDATE_ROOT/install.sh" --no-timer --snapshot-dir "$UP_STATE/package-rollback" +# Install root-owned config atomically. Never replace active DB credential values. +rclone_path=$(cat "$UP_STATE/rclone-path") +if [[ ! -e $rclone_path ]]; then + install -m 600 -o root -g root "$UP_STATE/rclone.conf" "$rclone_path" +else + [[ ! -L $rclone_path && $(stat -c %u "$rclone_path") == 0 ]] || die 'Unsafe live rclone configuration' + chmod 600 "$rclone_path" +fi +install -m 600 -o root -g root "$UP_STATE/candidate.conf" "$CONFIG_FILE.update-tmp" +mv -fT "$CONFIG_FILE.update-tmp" "$CONFIG_FILE" +load_config +build_plan +for file in "${DB_CNFS[@]}"; do secure_file "$file"; chmod 600 "$file"; done +# Hold the shared lock across every child. This also excludes manual restores. +bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --check +# Disable all retention in this acceptance run, including local retention. +bash /usr/local/sbin/app-backup.sh --config "$CONFIG_FILE" --no-retention +backup_run=$(python3 - "$WORKDIR/last-backup.json" <<'PY' +import json,sys +value=json.load(open(sys.argv[1])) +assert value['exit_code'] == 0 +print(value['run']) +PY +) +bash /usr/local/sbin/app-restore.sh --config "$CONFIG_FILE" --remote-run "$backup_run" --verify-only +printf '%s\n' "$backup_run" > "$UP_STATE/verified-backup-run" +# A missed persistent timer can fire immediately, so release the shared lock first. +unset APP_BACKUP_INHERIT_LOCK +flock -u 9 +systemctl enable --now app-backup.timer +UP_OK=true diff --git a/usr-local-sbin/app-backup.sh b/usr-local-sbin/app-backup.sh index 72a52f8..78e2b9a 100755 --- a/usr-local-sbin/app-backup.sh +++ b/usr-local-sbin/app-backup.sh @@ -8,11 +8,13 @@ LIB_DIR="$SCRIPT_DIR/../lib/app-backup" source "$LIB_DIR/common.sh" CONFIG_FILE=/etc/app-backup/app-backup.conf CHECK=false +SKIP_RETENTION=false while (( $# )); do case $1 in --config) CONFIG_FILE=${2:?Missing config path}; shift 2 ;; --check) CHECK=true; shift ;; - --help) echo 'Usage: app-backup.sh [--config FILE] [--check]'; exit 0 ;; + --no-retention) SKIP_RETENTION=true; shift ;; + --help) echo 'Usage: app-backup.sh [--config FILE] [--check] [--no-retention]'; exit 0 ;; *) die "Unknown argument: $1" ;; esac done @@ -99,6 +101,11 @@ if [[ $ENABLE_UPLOAD == true ]]; then echo '-- Remote backup fully read back and verified' fi +if [[ $SKIP_RETENTION == true ]]; then + echo "SUCCESS: $RUN_ID (retention skipped for acceptance run)" + exit 0 +fi + # Retention works on complete v2 sets only, after a new successful backup. # Older flat archives, partial runs and unknown folders are deliberately preserved. valid="$STAGING_DIR/local-retention.txt"