From d22838f818f0df4696154a80f1183818959f23f7 Mon Sep 17 00:00:00 2001 From: Johannes Rest Date: Sun, 18 Oct 2026 17:00:25 +0200 Subject: [PATCH] Preserve historical Gemini backups and separate snapshot staging --- SPEICHER.md | 61 ++++++ operations/gemini_backup_repair.py | 298 +++++++++++++++++++++++++++++ tests/test_gemini_backup_repair.py | 73 +++++++ 3 files changed, 432 insertions(+) create mode 100644 operations/gemini_backup_repair.py create mode 100644 tests/test_gemini_backup_repair.py diff --git a/SPEICHER.md b/SPEICHER.md index 8f75ff7..dfae66a 100644 --- a/SPEICHER.md +++ b/SPEICHER.md @@ -191,3 +191,64 @@ und `result.txt`. `SUCCESS` im aktuellen `result.txt` bestätigt das Update; der Restore-Teststatus ist ein separater Nachweis nach dem Monatslauf. Scheitert ein Update trotz der sicheren Bereinigung, bleiben die Timer nach begonnener Installation gestoppt; der Fehler muss behoben und das Paket erneut ausgeführt werden. + +## Gemini: historische Archive erhalten, Staging trennen – 18.10.2026 + +**Reparatur vorbereitet; produktive Ausführung und vollständiger neuer Lauf noch +offen.** Die durch Johannes ausgeführte Root-Diagnose bestätigt rund 48 GB +historische flache Archive sowie drei vollständige lokale Sätze ohne `UPLOADED`. +Die Platzprüfung des Nachtlaufs bricht vor dem Snapshot ab. Diese Daten werden +nicht als automatische Löschkandidaten freigegeben. + +`operations/gemini_backup_repair.py` ist eine feste Gemini-Operation. Sie wird +zusammen mit der ACME-Reparatur aus dem ai-rem-maintenance-Paket gestartet: + +```bash +sudo bash /var/tmp/install-gemini-repair-20261018.sh +sudo journalctl -fu jr-gemini-repair-20261018.service --no-pager +``` + +Der einmalige Hintergrundauftrag erfordert das Betreiberpasswort. Quellen, +Prüfsumme und vollständiger Ablauf stehen im +[Wartungsrepository](https://gitea.jr-it-services.de/johannes/ai-rem-maintenance). + +Die Speicheroperation erhält die bisherigen Archive wie folgt: + +- Historische flache Archive und bei Bedarf ältere vollständige, nicht als + hochgeladen bestätigte Sätze werden auf die freie Root-Partition unter + `/srv/app-backup-preserved-20261018/` **verlagert**. Quelle und Ziel jeder Datei + werden mit SHA-256 sowie rsync-Metadatenprüfung verglichen und Zielzustand + synchronisiert. Das ursprüngliche Objekt wird erst danach entfernt; die + Sicherung bleibt unter ihrem bisherigen Namen am neuen Ort erhalten. +- Bestätigte aktuelle lokale Sätze bleiben im aktiven Archivverzeichnis. Die + Verlagerung macht aus unbestätigten Sätzen keine bestätigten Uploads. Unter + `/var/lib/jr-app-backup-space-20261018/` liegen ursprüngliche Konfiguration, + Planung und ein Prüfnachweis mit Quell-/Zielort pro Objekt, ausschließlich root- + lesbar. Alte Pfade nach einer erfolgreichen Verlagerung nicht weiterverwenden. +- `STAGING_ROOT` wechselt nach erfolgreicher Prüfung auf + `/home/.app-backup-staging`; `ARCHIVE_DIR` bleibt unter `/var`. Neue und alte + Pfade dürfen keine Backup-Quellen einschließen oder in Quellen liegen. +- Die normale gemeinsame Backup-Sperre wird gehalten. Bestehende Recovery-Marker, + belegtes Staging, unbekannte Zielkopien, Symlinks, Hardlinks, fremde Besitzer, + beschreibbare Archive und zu wenig Platz führen zum Abbruch. Vor der + Konfigurationsaktivierung laufen die echten `--prepare-space`- und `--check`- + Prüfungen mit der vorgeschlagenen Konfiguration und geerbter echter Sperre. +- `MIN_FREE_GB`, Aufbewahrung und Schätzung bleiben unverändert. Auf `/` verbleiben + mindestens die konfigurierte Reserve plus 2 GiB. Auf `/var` wird Platz für das + zuletzt beobachtete Snapshotbudget plus Reserve plus 10 GiB zusätzlichen + Spielraum geschaffen. Die aktuelle Live-Schätzung ist trotzdem maßgeblich. + +Danach wird der normale `app-backup.service` gestartet. Der erfolgreiche +Reparaturauftrag bestätigt nur die Vorbereitung und das Einreihen dieses Laufs. +Erst dessen `SUCCESS` einschließlich Offsite-Prüfung bestätigt die Backup-Abnahme. +Der Kopier-/Dumpabschnitt hält wie bisher die konfigurierten Writer kurz an. + +Bei Teilfehlern bleiben verifizierte Zielkopien erhalten. Die alte Konfiguration +wird bis zum erfolgreichen Preflight nicht ersetzt. Eine Rücknahme der +Konfiguration allein stellt verlagerten Bestand und freien Platz nicht zurück; +kein blindes Zurückkopieren oder Löschen. Den Nachweis pro Objekt zur gezielten +Wiederherstellung beziehungsweise Auswahl historischer Restore-Quellen nutzen. + +Sieben lokale Tests prüfen Datei- und Verzeichnisverlagerung, Kopierkorruption +mit erhaltenem Original, Schutz vor Quellenüberschneidung/Links und Platzreserve. +Die produktive Abnahme und ein vollständiger Restore-Test stehen weiterhin aus. diff --git a/operations/gemini_backup_repair.py b/operations/gemini_backup_repair.py new file mode 100644 index 0000000..b0e390e --- /dev/null +++ b/operations/gemini_backup_repair.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Preserve old backup objects on /, use /home for staging, retain active /var archives.""" +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import socket +import stat +import subprocess +import tempfile + +CONFIG = Path('/etc/app-backup/app-backup.conf') +COMMON = Path('/usr/local/lib/app-backup/common.sh') +PRESERVED = Path('/srv/app-backup-preserved-20261018') +STATE = Path('/var/lib/jr-app-backup-space-20261018') +STAGING = Path('/home/.app-backup-staging') +GIB = 1024 ** 3 +BLOCK = '\n# Gemini: staging on a separate filesystem, 2026-10-18.\nSTAGING_ROOT="/home/.app-backup-staging"\n' + + +def checked(args, **kwargs): + print('+ ' + ' '.join(map(str, args)), flush=True) + return subprocess.run(list(map(str, args)), check=True, **kwargs) + + +def private_dir(path): + for item in [path, *path.parents]: + if item.is_symlink(): + raise RuntimeError('Symlink in working path: ' + str(item)) + path.mkdir(mode=0o700, exist_ok=True) + st = path.stat() + if st.st_uid != os.geteuid() or st.st_mode & 0o077: + raise RuntimeError('Working path is not private: ' + str(path)) + + +def disjoint(path, sources): + path = Path(path).resolve() + for src in sources: + src = Path(src).resolve() + if path == src or src in path.parents or path in src.parents: + raise RuntimeError('Backup source overlaps workspace: ' + str(src)) + + +def allocated(path): + paths = [path] + if path.is_dir(): + paths += list(path.rglob('*')) + total = 0 + for p in paths: + st = p.lstat() + if st.st_uid != os.geteuid() or st.st_mode & 0o022: + raise RuntimeError('Unexpected owner or writable archive: ' + str(p)) + if stat.S_ISLNK(st.st_mode) or not (stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode)): + raise RuntimeError('Unexpected link or special file: ' + str(p)) + if stat.S_ISREG(st.st_mode) and st.st_nlink != 1: + raise RuntimeError('Hard-linked archive is protected: ' + str(p)) + total += st.st_blocks * 512 + return total + + +def inventory(path): + allocated(path) # Refuse links/special objects before reading any contents. + paths = [path] if path.is_file() else sorted(p for p in path.rglob('*') if p.is_file()) + result = {} + for p in paths: + before = p.stat() + h = hashlib.sha256() + with p.open('rb') as f: + for block in iter(lambda: f.read(8 * 1024 * 1024), b''): + h.update(block) + after = p.stat() + if (before.st_size, before.st_mtime_ns, before.st_ino) != (after.st_size, after.st_mtime_ns, after.st_ino): + raise RuntimeError('Source changed during verification: ' + str(p)) + key = '.' if path.is_file() else str(p.relative_to(path)) + result[key] = {'bytes': before.st_size, 'sha256': h.hexdigest()} + return result + + +def sync_directory(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def json_atomic(path, data): + fd, tmp = tempfile.mkstemp(prefix='.state-', dir=path.parent) + try: + with os.fdopen(fd, 'w') as f: + json.dump(data, f, indent=2) + f.write('\n') + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + sync_directory(path.parent) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def migrate(source, destination, state_dir): + """Copy, validate bytes+metadata, then remove only this verified source object.""" + state_file = state_dir / (source.name + '.json') + if destination.is_symlink(): + raise RuntimeError('Destination is a symlink') + existed = state_file.exists() + if destination.exists() and not existed: + raise RuntimeError('Unrecognized preservation destination: ' + str(destination)) + print('Hash source: ' + str(source), flush=True) + expected = inventory(source) + record = {'source': str(source), 'destination': str(destination), 'files': expected} + if existed: + saved = json.loads(state_file.read_text()) + if any(saved[k] != record[k] for k in record): + raise RuntimeError('Source differs from previous preservation attempt') + json_atomic(state_file, record) + directory = source.is_dir() + if directory: + destination.mkdir(mode=0o700, exist_ok=True) + src = str(source) + ('/' if directory else '') + dst = str(destination) + ('/' if directory else '') + checked(['rsync', '-aHAX', '--numeric-ids', '--checksum', '--fsync', '--', src, dst]) + if inventory(destination) != expected or inventory(source) != expected: + raise RuntimeError('Byte comparison failed; original retained') + comparison = checked(['rsync', '-aHAXnci', '--numeric-ids', '--delete', '--', src, dst], + capture_output=True, text=True) + if comparison.stdout.strip(): + raise RuntimeError('Metadata/tree differs; original retained: ' + comparison.stdout) + for directory_path in ([p for p in destination.rglob('*') if p.is_dir()] + [destination] if directory else []): + sync_directory(directory_path) + sync_directory(destination.parent) + # Both copies have matching SHA-256 for every file; rsync also checked metadata. + record['verified'] = True + json_atomic(state_file, record) + if directory: + shutil.rmtree(source) + else: + source.unlink() + record['original_removed_after_verification'] = True + json_atomic(state_file, record) + print('Preserved and verified: ' + str(destination), flush=True) + + +def choose_moves(items, free_var, free_root, target_var, reserve_root): + selected = [] + for path, amount in items: + if free_var >= target_var: + break + if free_root - amount < reserve_root: + raise RuntimeError('Preservation would consume the root reserve') + selected.append((path, amount)) + free_var += amount + free_root -= amount + if free_var < target_var: + raise RuntimeError('Insufficient preservable data for a durable layout') + return selected + + +def plan(): + script = '''set -e +LIB_DIR=/usr/local/lib/app-backup +source "$1" +CONFIG_FILE=$2 +load_config +build_plan +guard_workspaces +printf '%s\\0' "$LOCKFILE" "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$MIN_FREE_GB" "$ARCHIVE_PREFIX" "${SOURCES[@]}" +''' + r = subprocess.run(['bash', '-c', script, 'plan', str(COMMON), str(CONFIG)], + check=True, capture_output=True) + fields = r.stdout.decode().rstrip('\0').split('\0') + if len(fields) < 7: + raise RuntimeError('Incomplete plan') + return fields[:6], fields[6:] + + +def free(path): + v = os.statvfs(path) + return v.f_bavail * v.f_frsize + + +def install_config(content): + fd, tmp = tempfile.mkstemp(prefix='.gemini-staging-', dir=CONFIG.parent) + try: + with os.fdopen(fd, 'wb') as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp, 0o600) + os.replace(tmp, CONFIG) + sync_directory(CONFIG.parent) + checked(['restorecon', CONFIG]) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def main(): + if os.geteuid() != 0 or socket.gethostname() not in ('gemini', 'gemini.jr-it-services.de'): + raise SystemExit('Run as root on Gemini only') + fields, sources = plan() + lock, work, old_staging, archives, reserve, prefix = fields + reserve = int(reserve) * GIB + archives = Path(archives) + if archives != Path('/var/backups/app-backup/archives') or prefix != 'appbackup': + raise SystemExit('Unexpected archive layout') + if list(Path(work).glob('recovery-*')): + raise SystemExit('Pending recovery state; operator must resolve first') + if Path(old_staging).exists() and any(Path(old_staging).iterdir()): + raise SystemExit('Existing staging data is protected; inspect first') + for path in (PRESERVED, STAGING, STATE): + disjoint(path, sources) + if not Path(lock).is_absolute(): + raise SystemExit('Unsafe lock path') + fd = os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_size or info.st_nlink != 1: + raise SystemExit('Unsafe backup lock') + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + if fd != 9: + os.dup2(fd, 9) + os.close(fd) + os.set_inheritable(9, True) + try: + locked_fields, locked_sources = plan() + if locked_fields != fields or locked_sources != sources: + raise RuntimeError("Backup configuration changed before lock acquisition") + for path in (PRESERVED, STAGING, STATE): + private_dir(path) + if STAGING.stat().st_dev == archives.stat().st_dev or PRESERVED.stat().st_dev == archives.stat().st_dev: + raise SystemExit('Required separate filesystems are absent') + # The unchanged real estimator is run again below before activating config. + journal = subprocess.run(['journalctl', '-u', 'app-backup.service', '--since', '2026-10-18', + '--no-pager', '-o', 'cat'], check=True, capture_output=True, text=True) + budgets = re.findall(r'Estimated snapshot budget: ([0-9]+) bytes', journal.stdout) + if not budgets: + raise SystemExit('No observed snapshot budget; inspect before relocation') + budget = int(budgets[-1]) + if free(STAGING) < budget + reserve: + raise SystemExit('Staging filesystem lacks observed budget plus unchanged reserve') + original = CONFIG.read_bytes() + saved = STATE / 'app-backup.conf.before' + if not saved.exists(): + if BLOCK.encode() in original: + raise SystemExit('Original config backup missing') + shutil.copy2(CONFIG, saved) + os.chmod(saved, 0o600) + elif original != saved.read_bytes() and BLOCK.encode() not in original: + raise SystemExit('Configuration changed since previous attempt') + files, directories = [], [] + for p in sorted(archives.iterdir()): + if p.is_symlink(): + continue + if p.is_file() and re.fullmatch(r'appbackup_[0-9_-]+_[a-z0-9-]+\.tar\.(zst|gz)', p.name): + files.append((p, allocated(p))) + elif p.is_dir() and (p/'COMPLETE').is_file() and (p/'manifest.json').is_file() and not (p/'UPLOADED').exists(): + directories.append((p, allocated(p))) + # Extra 10 GiB on /var covers source growth and retaining a newly uploaded + # local set; uploaded sets are never relocation candidates. + moves = choose_moves(files + directories, free(archives), free(PRESERVED), + budget + reserve + 10 * GIB, reserve + 2 * GIB) + json_atomic(STATE / 'relocation-plan.json', { + 'staging': str(STAGING), 'active_archives': str(archives), + 'preservation_root': str(PRESERVED), 'observed_snapshot_budget': budget, + 'moves': [{'source': str(p), 'allocated_bytes': n} for p, n in moves]}) + for src, amount in moves: + if free(PRESERVED) < amount + reserve + 2 * GIB: + raise RuntimeError('Root space changed; original object retained') + migrate(src, PRESERVED / src.name, STATE) + proposed = original if BLOCK.encode() in original else original.rstrip() + b'\n' + BLOCK.encode() + candidate = STATE / 'app-backup.conf.candidate' + candidate.write_bytes(proposed) + os.chmod(candidate, 0o600) + env = dict(os.environ, APP_BACKUP_INHERIT_LOCK='1') + for mode in ['--prepare-space', '--check']: + checked(['bash', '/usr/local/sbin/app-backup.sh', '--config', candidate, mode], + env=env, pass_fds=(9,)) + if CONFIG.read_bytes() != original: + raise RuntimeError('Backup config changed during validation') + if original != proposed: + try: + install_config(proposed) + except BaseException: + install_config(original) + raise + print('Storage layout validated; active archives remain on /var and all relocated bytes are retained.', flush=True) + finally: + os.close(9) + checked(['systemctl', 'start', '--no-block', 'app-backup.service']) + print('Full normal backup started. Final backup SUCCESS and offsite verification are still pending.', flush=True) + + +if __name__ == '__main__': + main() diff --git a/tests/test_gemini_backup_repair.py b/tests/test_gemini_backup_repair.py new file mode 100644 index 0000000..1c75086 --- /dev/null +++ b/tests/test_gemini_backup_repair.py @@ -0,0 +1,73 @@ +import importlib.util +from pathlib import Path +import shutil +import tempfile +import unittest +from unittest import mock + +spec = importlib.util.spec_from_file_location('repair', Path(__file__).resolve().parents[1] / 'operations/gemini_backup_repair.py') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + +class PreservationTests(unittest.TestCase): + def test_plan_keeps_root_reserve_and_stops_when_var_target_is_met(self): + self.assertEqual(m.choose_moves([('a', 48), ('b', 7), ('c', 7), ('d', 7)], 26, 77, 84, 12), [('a',48),('b',7),('c',7)]) + with self.assertRaises(RuntimeError): + m.choose_moves([('a', 70)], 26, 77, 84, 12) + with self.assertRaises(RuntimeError): + m.choose_moves([('a', 5)], 26, 77, 84, 12) + + def test_source_parent_child_overlap_is_rejected(self): + m.disjoint('/home/.staging', ['/home/johannes']) + for path,sources in [('/home/.staging',['/home']),('/srv/history',['/srv/history/source'])]: + with self.assertRaises(RuntimeError): + m.disjoint(path,sources) + + def test_symlink_and_hardlink_are_not_accepted_as_archive_data(self): + with tempfile.TemporaryDirectory() as t: + root=Path(t);source=root/'source';source.write_bytes(b'data') + link=root/'link';link.symlink_to(source) + with self.assertRaises(RuntimeError):m.inventory(link) + link.unlink();link.hardlink_to(source) + with self.assertRaises(RuntimeError):m.inventory(source) + + @unittest.skipUnless(shutil.which('rsync'), 'rsync required') + def test_copy_is_verified_before_original_is_removed(self): + with tempfile.TemporaryDirectory() as t: + root=Path(t);source=root/'archive';source.mkdir();(source/'data').write_bytes(b'saved backup') + state=root/'state';state.mkdir();dest=root/'preserved';dest.mkdir() + m.migrate(source,dest/'archive',state) + self.assertFalse(source.exists()) + self.assertEqual((dest/'archive/data').read_bytes(),b'saved backup') + + @unittest.skipUnless(shutil.which('rsync'), 'rsync required') + def test_flat_archive_is_preserved_with_its_original_filename(self): + with tempfile.TemporaryDirectory() as t: + root=Path(t);src=root/'old.tar.zst';src.write_bytes(b'old backup') + state=root/'state';state.mkdir();dstroot=root/'preserved';dstroot.mkdir() + m.migrate(src,dstroot/src.name,state) + self.assertFalse(src.exists()) + self.assertEqual((dstroot/src.name).read_bytes(),b'old backup') + + @unittest.skipUnless(shutil.which('rsync'), 'rsync required') + def test_corrupt_destination_keeps_original(self): + with tempfile.TemporaryDirectory() as t: + root=Path(t);src=root/'source';src.write_bytes(b'original backup') + state=root/'state';state.mkdir();dst=root/'destination' + real=m.checked + def corrupt_after_copy(args, **kw): + result=real(args, **kw) + if '--fsync' in args:dst.write_bytes(b'corrupted') + return result + with mock.patch.object(m,'checked',side_effect=corrupt_after_copy): + with self.assertRaises(RuntimeError):m.migrate(src,dst,state) + self.assertEqual(src.read_bytes(),b'original backup') + + def test_unrecognized_destination_never_removes_source(self): + with tempfile.TemporaryDirectory() as t: + root=Path(t);src=root/'source';src.write_bytes(b'original');dst=root/'destination';dst.write_bytes(b'unrelated') + state=root/'state';state.mkdir() + with self.assertRaises(RuntimeError):m.migrate(src,dst,state) + self.assertEqual(src.read_bytes(),b'original') + +if __name__ == '__main__':unittest.main()