Preserve historical Gemini backups and separate snapshot staging
This commit is contained in:
admin committed
1 parent
b3f140c254
commit
d22838f818
3 files changed
+432
No files matched your search
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserve old backup objects on /, use /home for staging, retain active /var archives."""
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
CONFIG = Path('/etc/app-backup/app-backup.conf')
|
||||
COMMON = Path('/usr/local/lib/app-backup/common.sh')
|
||||
PRESERVED = Path('/srv/app-backup-preserved-20261018')
|
||||
STATE = Path('/var/lib/jr-app-backup-space-20261018')
|
||||
STAGING = Path('/home/.app-backup-staging')
|
||||
GIB = 1024 ** 3
|
||||
BLOCK = '\n# Gemini: staging on a separate filesystem, 2026-10-18.\nSTAGING_ROOT="/home/.app-backup-staging"\n'
|
||||
|
||||
|
||||
def checked(args, **kwargs):
|
||||
print('+ ' + ' '.join(map(str, args)), flush=True)
|
||||
return subprocess.run(list(map(str, args)), check=True, **kwargs)
|
||||
|
||||
|
||||
def private_dir(path):
|
||||
for item in [path, *path.parents]:
|
||||
if item.is_symlink():
|
||||
raise RuntimeError('Symlink in working path: ' + str(item))
|
||||
path.mkdir(mode=0o700, exist_ok=True)
|
||||
st = path.stat()
|
||||
if st.st_uid != os.geteuid() or st.st_mode & 0o077:
|
||||
raise RuntimeError('Working path is not private: ' + str(path))
|
||||
|
||||
|
||||
def disjoint(path, sources):
|
||||
path = Path(path).resolve()
|
||||
for src in sources:
|
||||
src = Path(src).resolve()
|
||||
if path == src or src in path.parents or path in src.parents:
|
||||
raise RuntimeError('Backup source overlaps workspace: ' + str(src))
|
||||
|
||||
|
||||
def allocated(path):
|
||||
paths = [path]
|
||||
if path.is_dir():
|
||||
paths += list(path.rglob('*'))
|
||||
total = 0
|
||||
for p in paths:
|
||||
st = p.lstat()
|
||||
if st.st_uid != os.geteuid() or st.st_mode & 0o022:
|
||||
raise RuntimeError('Unexpected owner or writable archive: ' + str(p))
|
||||
if stat.S_ISLNK(st.st_mode) or not (stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode)):
|
||||
raise RuntimeError('Unexpected link or special file: ' + str(p))
|
||||
if stat.S_ISREG(st.st_mode) and st.st_nlink != 1:
|
||||
raise RuntimeError('Hard-linked archive is protected: ' + str(p))
|
||||
total += st.st_blocks * 512
|
||||
return total
|
||||
|
||||
|
||||
def inventory(path):
|
||||
allocated(path) # Refuse links/special objects before reading any contents.
|
||||
paths = [path] if path.is_file() else sorted(p for p in path.rglob('*') if p.is_file())
|
||||
result = {}
|
||||
for p in paths:
|
||||
before = p.stat()
|
||||
h = hashlib.sha256()
|
||||
with p.open('rb') as f:
|
||||
for block in iter(lambda: f.read(8 * 1024 * 1024), b''):
|
||||
h.update(block)
|
||||
after = p.stat()
|
||||
if (before.st_size, before.st_mtime_ns, before.st_ino) != (after.st_size, after.st_mtime_ns, after.st_ino):
|
||||
raise RuntimeError('Source changed during verification: ' + str(p))
|
||||
key = '.' if path.is_file() else str(p.relative_to(path))
|
||||
result[key] = {'bytes': before.st_size, 'sha256': h.hexdigest()}
|
||||
return result
|
||||
|
||||
|
||||
def sync_directory(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def json_atomic(path, data):
|
||||
fd, tmp = tempfile.mkstemp(prefix='.state-', dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.write('\n')
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, path)
|
||||
sync_directory(path.parent)
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
|
||||
|
||||
def migrate(source, destination, state_dir):
|
||||
"""Copy, validate bytes+metadata, then remove only this verified source object."""
|
||||
state_file = state_dir / (source.name + '.json')
|
||||
if destination.is_symlink():
|
||||
raise RuntimeError('Destination is a symlink')
|
||||
existed = state_file.exists()
|
||||
if destination.exists() and not existed:
|
||||
raise RuntimeError('Unrecognized preservation destination: ' + str(destination))
|
||||
print('Hash source: ' + str(source), flush=True)
|
||||
expected = inventory(source)
|
||||
record = {'source': str(source), 'destination': str(destination), 'files': expected}
|
||||
if existed:
|
||||
saved = json.loads(state_file.read_text())
|
||||
if any(saved[k] != record[k] for k in record):
|
||||
raise RuntimeError('Source differs from previous preservation attempt')
|
||||
json_atomic(state_file, record)
|
||||
directory = source.is_dir()
|
||||
if directory:
|
||||
destination.mkdir(mode=0o700, exist_ok=True)
|
||||
src = str(source) + ('/' if directory else '')
|
||||
dst = str(destination) + ('/' if directory else '')
|
||||
checked(['rsync', '-aHAX', '--numeric-ids', '--checksum', '--fsync', '--', src, dst])
|
||||
if inventory(destination) != expected or inventory(source) != expected:
|
||||
raise RuntimeError('Byte comparison failed; original retained')
|
||||
comparison = checked(['rsync', '-aHAXnci', '--numeric-ids', '--delete', '--', src, dst],
|
||||
capture_output=True, text=True)
|
||||
if comparison.stdout.strip():
|
||||
raise RuntimeError('Metadata/tree differs; original retained: ' + comparison.stdout)
|
||||
for directory_path in ([p for p in destination.rglob('*') if p.is_dir()] + [destination] if directory else []):
|
||||
sync_directory(directory_path)
|
||||
sync_directory(destination.parent)
|
||||
# Both copies have matching SHA-256 for every file; rsync also checked metadata.
|
||||
record['verified'] = True
|
||||
json_atomic(state_file, record)
|
||||
if directory:
|
||||
shutil.rmtree(source)
|
||||
else:
|
||||
source.unlink()
|
||||
record['original_removed_after_verification'] = True
|
||||
json_atomic(state_file, record)
|
||||
print('Preserved and verified: ' + str(destination), flush=True)
|
||||
|
||||
|
||||
def choose_moves(items, free_var, free_root, target_var, reserve_root):
|
||||
selected = []
|
||||
for path, amount in items:
|
||||
if free_var >= target_var:
|
||||
break
|
||||
if free_root - amount < reserve_root:
|
||||
raise RuntimeError('Preservation would consume the root reserve')
|
||||
selected.append((path, amount))
|
||||
free_var += amount
|
||||
free_root -= amount
|
||||
if free_var < target_var:
|
||||
raise RuntimeError('Insufficient preservable data for a durable layout')
|
||||
return selected
|
||||
|
||||
|
||||
def plan():
|
||||
script = '''set -e
|
||||
LIB_DIR=/usr/local/lib/app-backup
|
||||
source "$1"
|
||||
CONFIG_FILE=$2
|
||||
load_config
|
||||
build_plan
|
||||
guard_workspaces
|
||||
printf '%s\\0' "$LOCKFILE" "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$MIN_FREE_GB" "$ARCHIVE_PREFIX" "${SOURCES[@]}"
|
||||
'''
|
||||
r = subprocess.run(['bash', '-c', script, 'plan', str(COMMON), str(CONFIG)],
|
||||
check=True, capture_output=True)
|
||||
fields = r.stdout.decode().rstrip('\0').split('\0')
|
||||
if len(fields) < 7:
|
||||
raise RuntimeError('Incomplete plan')
|
||||
return fields[:6], fields[6:]
|
||||
|
||||
|
||||
def free(path):
|
||||
v = os.statvfs(path)
|
||||
return v.f_bavail * v.f_frsize
|
||||
|
||||
|
||||
def install_config(content):
|
||||
fd, tmp = tempfile.mkstemp(prefix='.gemini-staging-', dir=CONFIG.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.chmod(tmp, 0o600)
|
||||
os.replace(tmp, CONFIG)
|
||||
sync_directory(CONFIG.parent)
|
||||
checked(['restorecon', CONFIG])
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
|
||||
|
||||
def main():
|
||||
if os.geteuid() != 0 or socket.gethostname() not in ('gemini', 'gemini.jr-it-services.de'):
|
||||
raise SystemExit('Run as root on Gemini only')
|
||||
fields, sources = plan()
|
||||
lock, work, old_staging, archives, reserve, prefix = fields
|
||||
reserve = int(reserve) * GIB
|
||||
archives = Path(archives)
|
||||
if archives != Path('/var/backups/app-backup/archives') or prefix != 'appbackup':
|
||||
raise SystemExit('Unexpected archive layout')
|
||||
if list(Path(work).glob('recovery-*')):
|
||||
raise SystemExit('Pending recovery state; operator must resolve first')
|
||||
if Path(old_staging).exists() and any(Path(old_staging).iterdir()):
|
||||
raise SystemExit('Existing staging data is protected; inspect first')
|
||||
for path in (PRESERVED, STAGING, STATE):
|
||||
disjoint(path, sources)
|
||||
if not Path(lock).is_absolute():
|
||||
raise SystemExit('Unsafe lock path')
|
||||
fd = os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_size or info.st_nlink != 1:
|
||||
raise SystemExit('Unsafe backup lock')
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if fd != 9:
|
||||
os.dup2(fd, 9)
|
||||
os.close(fd)
|
||||
os.set_inheritable(9, True)
|
||||
try:
|
||||
locked_fields, locked_sources = plan()
|
||||
if locked_fields != fields or locked_sources != sources:
|
||||
raise RuntimeError("Backup configuration changed before lock acquisition")
|
||||
for path in (PRESERVED, STAGING, STATE):
|
||||
private_dir(path)
|
||||
if STAGING.stat().st_dev == archives.stat().st_dev or PRESERVED.stat().st_dev == archives.stat().st_dev:
|
||||
raise SystemExit('Required separate filesystems are absent')
|
||||
# The unchanged real estimator is run again below before activating config.
|
||||
journal = subprocess.run(['journalctl', '-u', 'app-backup.service', '--since', '2026-10-18',
|
||||
'--no-pager', '-o', 'cat'], check=True, capture_output=True, text=True)
|
||||
budgets = re.findall(r'Estimated snapshot budget: ([0-9]+) bytes', journal.stdout)
|
||||
if not budgets:
|
||||
raise SystemExit('No observed snapshot budget; inspect before relocation')
|
||||
budget = int(budgets[-1])
|
||||
if free(STAGING) < budget + reserve:
|
||||
raise SystemExit('Staging filesystem lacks observed budget plus unchanged reserve')
|
||||
original = CONFIG.read_bytes()
|
||||
saved = STATE / 'app-backup.conf.before'
|
||||
if not saved.exists():
|
||||
if BLOCK.encode() in original:
|
||||
raise SystemExit('Original config backup missing')
|
||||
shutil.copy2(CONFIG, saved)
|
||||
os.chmod(saved, 0o600)
|
||||
elif original != saved.read_bytes() and BLOCK.encode() not in original:
|
||||
raise SystemExit('Configuration changed since previous attempt')
|
||||
files, directories = [], []
|
||||
for p in sorted(archives.iterdir()):
|
||||
if p.is_symlink():
|
||||
continue
|
||||
if p.is_file() and re.fullmatch(r'appbackup_[0-9_-]+_[a-z0-9-]+\.tar\.(zst|gz)', p.name):
|
||||
files.append((p, allocated(p)))
|
||||
elif p.is_dir() and (p/'COMPLETE').is_file() and (p/'manifest.json').is_file() and not (p/'UPLOADED').exists():
|
||||
directories.append((p, allocated(p)))
|
||||
# Extra 10 GiB on /var covers source growth and retaining a newly uploaded
|
||||
# local set; uploaded sets are never relocation candidates.
|
||||
moves = choose_moves(files + directories, free(archives), free(PRESERVED),
|
||||
budget + reserve + 10 * GIB, reserve + 2 * GIB)
|
||||
json_atomic(STATE / 'relocation-plan.json', {
|
||||
'staging': str(STAGING), 'active_archives': str(archives),
|
||||
'preservation_root': str(PRESERVED), 'observed_snapshot_budget': budget,
|
||||
'moves': [{'source': str(p), 'allocated_bytes': n} for p, n in moves]})
|
||||
for src, amount in moves:
|
||||
if free(PRESERVED) < amount + reserve + 2 * GIB:
|
||||
raise RuntimeError('Root space changed; original object retained')
|
||||
migrate(src, PRESERVED / src.name, STATE)
|
||||
proposed = original if BLOCK.encode() in original else original.rstrip() + b'\n' + BLOCK.encode()
|
||||
candidate = STATE / 'app-backup.conf.candidate'
|
||||
candidate.write_bytes(proposed)
|
||||
os.chmod(candidate, 0o600)
|
||||
env = dict(os.environ, APP_BACKUP_INHERIT_LOCK='1')
|
||||
for mode in ['--prepare-space', '--check']:
|
||||
checked(['bash', '/usr/local/sbin/app-backup.sh', '--config', candidate, mode],
|
||||
env=env, pass_fds=(9,))
|
||||
if CONFIG.read_bytes() != original:
|
||||
raise RuntimeError('Backup config changed during validation')
|
||||
if original != proposed:
|
||||
try:
|
||||
install_config(proposed)
|
||||
except BaseException:
|
||||
install_config(original)
|
||||
raise
|
||||
print('Storage layout validated; active archives remain on /var and all relocated bytes are retained.', flush=True)
|
||||
finally:
|
||||
os.close(9)
|
||||
checked(['systemctl', 'start', '--no-block', 'app-backup.service'])
|
||||
print('Full normal backup started. Final backup SUCCESS and offsite verification are still pending.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user