349 lines
14 KiB
Python
349 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Preserve old backup objects on /, use /home for staging, retain active /var archives."""
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
|
|
CONFIG = Path('/etc/app-backup/app-backup.conf')
|
|
COMMON = Path('/usr/local/lib/app-backup/common.sh')
|
|
PRESERVED = Path('/srv/app-backup-preserved-20261018')
|
|
STATE = Path('/var/lib/jr-app-backup-space-20261018')
|
|
STAGING = Path('/home/.app-backup-staging')
|
|
GIB = 1024 ** 3
|
|
BLOCK = '\n# Gemini: staging on a separate filesystem, 2026-10-18.\nSTAGING_ROOT="/home/.app-backup-staging"\n'
|
|
|
|
|
|
def checked(args, **kwargs):
|
|
print('+ ' + ' '.join(map(str, args)), flush=True)
|
|
return subprocess.run(list(map(str, args)), check=True, **kwargs)
|
|
|
|
|
|
def private_dir(path):
|
|
for item in [path, *path.parents]:
|
|
if item.is_symlink():
|
|
raise RuntimeError('Symlink in working path: ' + str(item))
|
|
path.mkdir(mode=0o700, exist_ok=True)
|
|
st = path.stat()
|
|
if st.st_uid != os.geteuid() or st.st_mode & 0o077:
|
|
raise RuntimeError('Working path is not private: ' + str(path))
|
|
|
|
|
|
def disjoint(path, sources):
|
|
path = Path(path).resolve()
|
|
for src in sources:
|
|
src = Path(src).resolve()
|
|
if path == src or src in path.parents or path in src.parents:
|
|
raise RuntimeError('Backup source overlaps workspace: ' + str(src))
|
|
|
|
|
|
def allocated(path):
|
|
paths = [path]
|
|
if path.is_dir():
|
|
paths += list(path.rglob('*'))
|
|
total = 0
|
|
for p in paths:
|
|
st = p.lstat()
|
|
if st.st_uid != os.geteuid() or st.st_mode & 0o022:
|
|
raise RuntimeError('Unexpected owner or writable archive: ' + str(p))
|
|
if stat.S_ISLNK(st.st_mode) or not (stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode)):
|
|
raise RuntimeError('Unexpected link or special file: ' + str(p))
|
|
if stat.S_ISREG(st.st_mode) and st.st_nlink != 1:
|
|
raise RuntimeError('Hard-linked archive is protected: ' + str(p))
|
|
total += st.st_blocks * 512
|
|
return total
|
|
|
|
|
|
def inventory(path):
|
|
allocated(path) # Refuse links/special objects before reading any contents.
|
|
paths = [path] if path.is_file() else sorted(p for p in path.rglob('*') if p.is_file())
|
|
result = {}
|
|
for p in paths:
|
|
before = p.stat()
|
|
h = hashlib.sha256()
|
|
with p.open('rb') as f:
|
|
for block in iter(lambda: f.read(8 * 1024 * 1024), b''):
|
|
h.update(block)
|
|
after = p.stat()
|
|
if (before.st_size, before.st_mtime_ns, before.st_ino) != (after.st_size, after.st_mtime_ns, after.st_ino):
|
|
raise RuntimeError('Source changed during verification: ' + str(p))
|
|
key = '.' if path.is_file() else str(p.relative_to(path))
|
|
result[key] = {'bytes': before.st_size, 'sha256': h.hexdigest()}
|
|
return result
|
|
|
|
|
|
def sync_directory(path):
|
|
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def json_atomic(path, data):
|
|
fd, tmp = tempfile.mkstemp(prefix='.state-', dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, 'w') as f:
|
|
json.dump(data, f, indent=2)
|
|
f.write('\n')
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.replace(tmp, path)
|
|
sync_directory(path.parent)
|
|
finally:
|
|
if os.path.exists(tmp):
|
|
os.unlink(tmp)
|
|
|
|
|
|
def migrate(source, destination, state_dir):
|
|
"""Copy, validate bytes+metadata, then remove only this verified source object."""
|
|
state_file = state_dir / (source.name + '.json')
|
|
if destination.is_symlink():
|
|
raise RuntimeError('Destination is a symlink')
|
|
existed = state_file.exists()
|
|
if destination.exists() and not existed:
|
|
raise RuntimeError('Unrecognized preservation destination: ' + str(destination))
|
|
print('Hash source: ' + str(source), flush=True)
|
|
expected = inventory(source)
|
|
record = {'source': str(source), 'destination': str(destination), 'files': expected}
|
|
if existed:
|
|
saved = json.loads(state_file.read_text())
|
|
if any(saved[k] != record[k] for k in record):
|
|
raise RuntimeError('Source differs from previous preservation attempt')
|
|
json_atomic(state_file, record)
|
|
directory = source.is_dir()
|
|
if directory:
|
|
destination.mkdir(mode=0o700, exist_ok=True)
|
|
src = str(source) + ('/' if directory else '')
|
|
dst = str(destination) + ('/' if directory else '')
|
|
checked(['rsync', '-aHAX', '--numeric-ids', '--checksum', '--fsync', '--', src, dst])
|
|
if inventory(destination) != expected or inventory(source) != expected:
|
|
raise RuntimeError('Byte comparison failed; original retained')
|
|
comparison = checked(['rsync', '-aHAXnci', '--numeric-ids', '--delete', '--', src, dst],
|
|
capture_output=True, text=True)
|
|
if comparison.stdout.strip():
|
|
raise RuntimeError('Metadata/tree differs; original retained: ' + comparison.stdout)
|
|
for directory_path in ([p for p in destination.rglob('*') if p.is_dir()] + [destination] if directory else []):
|
|
sync_directory(directory_path)
|
|
sync_directory(destination.parent)
|
|
# Both copies have matching SHA-256 for every file; rsync also checked metadata.
|
|
record['verified'] = True
|
|
json_atomic(state_file, record)
|
|
if directory:
|
|
shutil.rmtree(source)
|
|
else:
|
|
source.unlink()
|
|
record['original_removed_after_verification'] = True
|
|
json_atomic(state_file, record)
|
|
print('Preserved and verified: ' + str(destination), flush=True)
|
|
|
|
|
|
def choose_moves(items, free_var, free_root, target_var, reserve_root):
|
|
selected = []
|
|
for path, amount in items:
|
|
if free_var >= target_var:
|
|
break
|
|
if free_root - amount < reserve_root:
|
|
raise RuntimeError('Preservation would consume the root reserve')
|
|
selected.append((path, amount))
|
|
free_var += amount
|
|
free_root -= amount
|
|
if free_var < target_var:
|
|
raise RuntimeError('Insufficient preservable data for a durable layout')
|
|
return selected
|
|
|
|
|
|
def plan():
|
|
script = '''set -e
|
|
LIB_DIR=/usr/local/lib/app-backup
|
|
source "$1"
|
|
CONFIG_FILE=$2
|
|
load_config
|
|
build_plan
|
|
guard_workspaces
|
|
printf '%s\\0' "$LOCKFILE" "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$MIN_FREE_GB" "$ARCHIVE_PREFIX" "${SOURCES[@]}"
|
|
'''
|
|
r = subprocess.run(['bash', '-c', script, 'plan', str(COMMON), str(CONFIG)],
|
|
check=True, capture_output=True)
|
|
fields = r.stdout.decode().rstrip('\0').split('\0')
|
|
if len(fields) < 7:
|
|
raise RuntimeError('Incomplete plan')
|
|
return fields[:6], fields[6:]
|
|
|
|
|
|
def snapshot_budget(file_stats, database_sizes):
|
|
total = 0
|
|
for size, count in file_stats:
|
|
if size < 0 or count < 0:
|
|
raise ValueError('Invalid file statistics')
|
|
total += size + count * 8192
|
|
for size in database_sizes:
|
|
if size < 0:
|
|
raise ValueError('Invalid database size')
|
|
total += size * 3 + 67108864
|
|
return total * 12 // 10 + 67108864
|
|
|
|
|
|
def estimate_budget():
|
|
# Same dry-run exclusions, file metadata, SQL expansion and growth allowance
|
|
# as installed lib/space.sh. No cleanup, write, service stop or journal needed.
|
|
script = r'''set -e
|
|
LIB_DIR=/usr/local/lib/app-backup
|
|
source "$1"
|
|
CONFIG_FILE=$2
|
|
load_config
|
|
build_plan
|
|
guard_workspaces
|
|
for i in "${!LABELS[@]}"; do
|
|
check_source_path "${SOURCES[i]}"
|
|
exclusions "${LABELS[i]}"
|
|
stats=$(rsync -aHAX --dry-run --stats --numeric-ids "${EXCLUDES[@]}" -- "${SOURCES[i]}/" "$STAGING_ROOT/.space-estimate/" 8>&- 9>&-)
|
|
bytes=$(awk '/^Total file size:/ {gsub(/,/, "", $4); print $4}' <<< "$stats")
|
|
count=$(awk '/^Number of files:/ {gsub(/,/, "", $4); print $4}' <<< "$stats")
|
|
[[ $bytes =~ ^[0-9]+$ && $count =~ ^[0-9]+$ ]] || exit 1
|
|
printf 'F %s %s\n' "$bytes" "$count"
|
|
done
|
|
for i in "${!DB_NAMES[@]}"; do
|
|
secure_file "${DB_CNFS[i]}"
|
|
size=$("$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --batch --skip-column-names --execute='SELECT COALESCE(SUM(data_length+index_length),0) FROM information_schema.tables WHERE table_schema=DATABASE()')
|
|
[[ $size =~ ^[0-9]+$ ]] || exit 1
|
|
printf 'D %s\n' "$size"
|
|
done
|
|
'''
|
|
result = subprocess.run(['bash', '-c', script, 'estimate', str(COMMON), str(CONFIG)],
|
|
check=True, capture_output=True, text=True, pass_fds=(9,), timeout=900)
|
|
files, databases = [], []
|
|
for line in result.stdout.splitlines():
|
|
if re.fullmatch(r'F [0-9]+ [0-9]+', line):
|
|
_, size, count = line.split()
|
|
files.append((int(size), int(count)))
|
|
elif re.fullmatch(r'D [0-9]+', line):
|
|
databases.append(int(line.split()[1]))
|
|
else:
|
|
raise RuntimeError('Unexpected source estimation output')
|
|
if not files:
|
|
raise RuntimeError('No source file estimates')
|
|
return snapshot_budget(files, databases)
|
|
|
|
|
|
def free(path):
|
|
v = os.statvfs(path)
|
|
return v.f_bavail * v.f_frsize
|
|
|
|
|
|
def install_config(content):
|
|
fd, tmp = tempfile.mkstemp(prefix='.gemini-staging-', dir=CONFIG.parent)
|
|
try:
|
|
with os.fdopen(fd, 'wb') as f:
|
|
f.write(content)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.chmod(tmp, 0o600)
|
|
os.replace(tmp, CONFIG)
|
|
sync_directory(CONFIG.parent)
|
|
checked(['restorecon', CONFIG])
|
|
finally:
|
|
if os.path.exists(tmp):
|
|
os.unlink(tmp)
|
|
|
|
|
|
def main():
|
|
if os.geteuid() != 0 or socket.gethostname() not in ('gemini', 'gemini.jr-it-services.de'):
|
|
raise SystemExit('Run as root on Gemini only')
|
|
fields, sources = plan()
|
|
lock, work, old_staging, archives, reserve, prefix = fields
|
|
reserve = int(reserve) * GIB
|
|
archives = Path(archives)
|
|
if archives != Path('/var/backups/app-backup/archives') or prefix != 'appbackup':
|
|
raise SystemExit('Unexpected archive layout')
|
|
if list(Path(work).glob('recovery-*')):
|
|
raise SystemExit('Pending recovery state; operator must resolve first')
|
|
if Path(old_staging).exists() and any(Path(old_staging).iterdir()):
|
|
raise SystemExit('Existing staging data is protected; inspect first')
|
|
for path in (PRESERVED, STAGING, STATE):
|
|
disjoint(path, sources)
|
|
if not Path(lock).is_absolute():
|
|
raise SystemExit('Unsafe lock path')
|
|
fd = os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
info = os.fstat(fd)
|
|
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_size or info.st_nlink != 1:
|
|
raise SystemExit('Unsafe backup lock')
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if fd != 9:
|
|
os.dup2(fd, 9)
|
|
os.close(fd)
|
|
os.set_inheritable(9, True)
|
|
try:
|
|
locked_fields, locked_sources = plan()
|
|
if locked_fields != fields or locked_sources != sources:
|
|
raise RuntimeError("Backup configuration changed before lock acquisition")
|
|
for path in (PRESERVED, STAGING, STATE):
|
|
private_dir(path)
|
|
if STAGING.stat().st_dev == archives.stat().st_dev or PRESERVED.stat().st_dev == archives.stat().st_dev:
|
|
raise SystemExit('Required separate filesystems are absent')
|
|
budget = estimate_budget()
|
|
print('Live snapshot budget: ' + str(budget) + ' bytes', flush=True)
|
|
if free(STAGING) < budget + reserve:
|
|
raise SystemExit('Staging filesystem lacks live budget plus unchanged reserve')
|
|
original = CONFIG.read_bytes()
|
|
saved = STATE / 'app-backup.conf.before'
|
|
if not saved.exists():
|
|
if BLOCK.encode() in original:
|
|
raise SystemExit('Original config backup missing')
|
|
shutil.copy2(CONFIG, saved)
|
|
os.chmod(saved, 0o600)
|
|
elif original != saved.read_bytes() and BLOCK.encode() not in original:
|
|
raise SystemExit('Configuration changed since previous attempt')
|
|
files, directories = [], []
|
|
for p in sorted(archives.iterdir()):
|
|
if p.is_symlink():
|
|
continue
|
|
if p.is_file() and re.fullmatch(r'appbackup_[0-9_-]+_[a-z0-9-]+\.tar\.(zst|gz)', p.name):
|
|
files.append((p, allocated(p)))
|
|
elif p.is_dir() and (p/'COMPLETE').is_file() and (p/'manifest.json').is_file() and not (p/'UPLOADED').exists():
|
|
directories.append((p, allocated(p)))
|
|
# Extra 10 GiB on /var covers source growth and retaining a newly uploaded
|
|
# local set; uploaded sets are never relocation candidates.
|
|
moves = choose_moves(files + directories, free(archives), free(PRESERVED),
|
|
budget + reserve + 10 * GIB, reserve + 2 * GIB)
|
|
json_atomic(STATE / 'relocation-plan.json', {
|
|
'staging': str(STAGING), 'active_archives': str(archives),
|
|
'preservation_root': str(PRESERVED), 'live_snapshot_budget': budget,
|
|
'moves': [{'source': str(p), 'allocated_bytes': n} for p, n in moves]})
|
|
for src, amount in moves:
|
|
if free(PRESERVED) < amount + reserve + 2 * GIB:
|
|
raise RuntimeError('Root space changed; original object retained')
|
|
migrate(src, PRESERVED / src.name, STATE)
|
|
proposed = original if BLOCK.encode() in original else original.rstrip() + b'\n' + BLOCK.encode()
|
|
candidate = STATE / 'app-backup.conf.candidate'
|
|
candidate.write_bytes(proposed)
|
|
os.chmod(candidate, 0o600)
|
|
env = dict(os.environ, APP_BACKUP_INHERIT_LOCK='1')
|
|
for mode in ['--prepare-space', '--check']:
|
|
checked(['bash', '/usr/local/sbin/app-backup.sh', '--config', candidate, mode],
|
|
env=env, pass_fds=(9,))
|
|
if CONFIG.read_bytes() != original:
|
|
raise RuntimeError('Backup config changed during validation')
|
|
if original != proposed:
|
|
try:
|
|
install_config(proposed)
|
|
except BaseException:
|
|
install_config(original)
|
|
raise
|
|
print('Storage layout validated; active archives remain on /var and all relocated bytes are retained.', flush=True)
|
|
finally:
|
|
os.close(9)
|
|
checked(['systemctl', 'start', '--no-block', 'app-backup.service'])
|
|
print('Full normal backup started. Final backup SUCCESS and offsite verification are still pending.', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|