johannes/app-backup
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Recover backup capacity with verified adaptive retention

Browse Source 
Resume pending publications under local pressure, allow verified offsite-only history as a final local recovery stage, and prune old complete remote sets against actual upload quota while preserving the newest verified copies. Document defaults, recovery limits and validation; add pressure and quota regression coverage.
This commit is contained in:
admin committed 2026-10-22 15:08:35 +02:00
1 parent a79f48bc2a
commit 6737405b35
10 files changed
+601 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/test_backup.py
+190 -1
View File
@@ -340,7 +340,7 @@ QUIESCE_SERVICES=(nc-cron)
(path/'manifest.json').write_text(json.dumps(manifest))
(path/'COMPLETE').write_text('APP_BACKUP_V2 '+HELPER.digest(path/'manifest.json')+'\n')
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'copy', str(path), 'crypt:sets/'+path.name], check=True, capture_output=True)
'copy', str(path), 'crypt:sets/'+path.name, '--exclude', '/UPLOADED'], check=True, capture_output=True)
copies.append(path)
self.env['TEST_METADATA_ONLY'] = 'true'
self.events.write_text('')
@@ -371,6 +371,7 @@ QUIESCE_SERVICES=(nc-cron)
def test_pressure_keeps_last_uploaded_and_unconfirmed_copies(self):
current, copies = self.pressure_copies()
(copies[0]/'UPLOADED').unlink()
(copies[0]/'wordpress.tar.zst').write_bytes(b'invalid unconfirmed set')
self.fake_pressure(copies[0], always=True)
result = self.run_script('app-backup.sh', '--prepare-space', ok=False)
self.assertTrue(current.exists())
@@ -428,6 +429,194 @@ QUIESCE_SERVICES=(nc-cron)
result = self.run_script('app-backup.sh', '--check', ok=False)
self.assertIn('SPACE_KEEP_LOCAL must', result.stdout)
def space_call(self, command, ok=True):
self.events.write_text('') # Observe maintenance only, excluding setup backup.
result = subprocess.run(['bash', '-c', '''
LIB_DIR=$1/lib
source "$LIB_DIR/common.sh"
CONFIG_FILE=$2
OPERATION=backup
CHECK=true
load_config
init_runtime
build_plan
source "$1/lib/space.sh"
clean_backup_scratch
check_remote
eval "$3"
''', 'space-test', str(REPO), str(self.config), command], env=self.env,
text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=90)
if ok: self.assertEqual(result.returncode, 0, result.stdout)
else: self.assertNotEqual(result.returncode, 0, result.stdout)
return result
def remote_names(self):
return subprocess.check_output(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'lsf', 'crypt:sets', '--dirs-only'], text=True)
def quota_wrapper(self, mode='recover'):
wrapper = self.bin/'rclone'
wrapper.write_text('''#!/bin/bash
if [[ " $* " == *' about '* ]]; then
if [[ $TEST_QUOTA_MODE == unavailable ]]; then exit 9; fi
if [[ $TEST_QUOTA_MODE == cancel ]]; then exit 143; fi
if [[ $TEST_QUOTA_MODE == small ]]; then echo '{"free":2000000}'; exit 0; fi
if [[ $TEST_QUOTA_MODE == malformed ]]; then echo '{"free":true}'; exit 0; fi
if [[ $TEST_QUOTA_MODE == recover && -f $TEST_EVENTS.freed ]]; then
echo '{"free":1000000000000}'
else echo '{"free":0}'; fi
exit 0
fi
if [[ " $* " == *' purge '* ]]; then
/usr/bin/rclone "$@" || exit $?
touch "$TEST_EVENTS.freed"
exit 0
fi
exec /usr/bin/rclone "$@"
''')
wrapper.chmod(0o700)
self.env['TEST_QUOTA_MODE'] = mode
def test_pressure_releases_last_local_copy_and_next_backup_succeeds(self):
self.crypt_config()
current = self.backup()
# Real planning budget, but only a one-byte deficit. Releasing the last
# uploaded set solves preflight, then a real new backup/upload completes.
planning = self.run_script('app-backup.sh', '--prepare-space')
import re
budget = int(re.search(r'Estimated snapshot budget: (\d+)', planning.stdout)[1]) * 2
fake = self.bin/'df'
fake.write_text('#!/usr/bin/env python3\nfrom pathlib import Path\n'
f'available = {budget-1} if Path({str(current)!r}).exists() else 10**12\n'
'print("Filesystem 1B-blocks Used Available Use% Mounted")\n'
'print(f"root 2000000000000 0 {available} 0% /")\n')
fake.chmod(0o700)
result = self.run_script('app-backup.sh')
self.assertIn('Emergency space recovery: removed local copy', result.stdout)
self.assertIn('SUCCESS:', result.stdout)
self.assertFalse(current.exists())
self.assertIn(current.name, self.remote_names())
new = list(current.parent.glob('appbackup_*'))
self.assertEqual(len(new), 1)
self.assertTrue((new[0]/'UPLOADED').is_file())
self.assertEqual(json.loads(self.state.read_text()), self.initial)
def test_remote_only_can_be_disabled(self):
self.crypt_config()
current = self.backup()
with self.config.open('a') as f: f.write('SPACE_ALLOW_REMOTE_ONLY=false\n')
self.fake_pressure(current)
self.space_call('ensure_space "$ARCHIVE_DIR" 1000', ok=False)
self.assertTrue(current.exists())
def test_last_local_copy_is_kept_if_offsite_identity_is_wrong(self):
self.crypt_config()
current = self.backup()
original = (current/'COMPLETE').read_bytes()
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'rcat', 'crypt:sets/'+current.name+'/COMPLETE'],
input=b'X'+original[1:], check=True, capture_output=True)
self.fake_pressure(current)
result = self.space_call('ensure_space "$ARCHIVE_DIR" 1000', ok=False)
self.assertTrue(current.exists())
self.assertIn('Emergency cleanup keeps', result.stdout)
self.assertNotIn('systemctl stop', self.events.read_text())
def test_last_local_copy_is_kept_if_it_cannot_resolve_deficit(self):
self.crypt_config()
current = self.backup()
self.fake_pressure(current)
self.space_call('ensure_space "$ARCHIVE_DIR" 1000000000000', ok=False)
self.assertTrue(current.exists())
def test_pressure_retries_failed_upload_before_releasing_local_set(self):
self.crypt_config()
current = self.backup()
(current/'UPLOADED').unlink()
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'purge', 'crypt:sets/'+current.name], check=True, capture_output=True)
self.fake_pressure(current)
result = self.space_call('ensure_space "$ARCHIVE_DIR" 1000')
self.assertIn('Pending publication confirmed', result.stdout)
self.assertFalse(current.exists())
self.assertIn(current.name, self.remote_names())
def test_pending_complete_upload_reconciles_without_spare_remote_quota(self):
self.crypt_config()
current = self.backup()
(current/'UPLOADED').unlink()
self.fake_pressure(current)
self.quota_wrapper('stuck')
result = self.space_call('ensure_space "$ARCHIVE_DIR" 1000')
self.assertIn('Pending publication already complete offsite', result.stdout)
self.assertNotIn('Remote pressure cleanup: removing', result.stdout)
self.assertFalse(current.exists())
self.assertIn(current.name, self.remote_names())
def test_remote_quota_uses_compressed_archives_instead_of_snapshot_budget(self):
self.crypt_config()
self.quota_wrapper('small')
result = self.run_script('app-backup.sh')
self.assertIn('SUCCESS:', result.stdout)
self.assertNotIn('Remote pressure cleanup: removing', result.stdout)
self.assertTrue(any((p/'UPLOADED').exists() for p in (self.base/'work/archives').iterdir()))
def test_remote_pressure_deletes_oldest_only_and_keeps_newest_and_partial(self):
current, copies = self.pressure_copies()
partial = 'appbackup_2019-01-01_00-00-00_aaaaaaaa'
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'rcat', 'crypt:sets/'+partial+'/manifest.json'],
input=b'not complete', check=True, capture_output=True)
self.quota_wrapper()
result = self.space_call('ensure_remote_space 1000')
self.assertNotIn(copies[0].name, self.remote_names())
self.assertIn(copies[1].name, self.remote_names())
self.assertIn(current.name, self.remote_names())
self.assertIn(partial, self.remote_names())
self.assertTrue(all(p.exists() for p in [current]+copies))
self.assertIn('preserves unverified/incomplete', result.stdout)
self.assertNotIn('forbidden-archive-read', self.events.read_text())
def test_remote_pressure_stops_at_one_when_quota_does_not_recover(self):
current, copies = self.pressure_copies()
self.quota_wrapper('stuck')
result = self.space_call('ensure_remote_space 1000', ok=False)
self.assertIn(current.name, self.remote_names())
self.assertTrue(all(p.name not in self.remote_names() for p in copies))
self.assertIn('Insufficient remote quota', result.stdout)
self.assertTrue(current.exists())
def test_remote_pressure_rejects_size_corruption_and_directory_identity_mismatch(self):
current, copies = self.pressure_copies()
# Latest must be a valid survivor. Invalid ones never count or get purged.
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'deletefile', 'crypt:sets/'+current.name+'/wordpress.tar.zst'],
check=True, capture_output=True)
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'copyto', 'crypt:sets/'+copies[0].name+'/manifest.json',
'crypt:sets/'+copies[1].name+'/manifest.json'], check=True, capture_output=True)
subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
'copyto', 'crypt:sets/'+copies[0].name+'/COMPLETE',
'crypt:sets/'+copies[1].name+'/COMPLETE'], check=True, capture_output=True)
self.quota_wrapper()
self.space_call('ensure_remote_space 1000', ok=False)
self.assertTrue(all(p.name in self.remote_names() for p in [current]+copies))
def test_remote_quota_unavailable_or_malformed_never_deletes(self):
current, copies = self.pressure_copies()
for mode in ('unavailable', 'malformed'):
self.quota_wrapper(mode)
result = self.space_call('ensure_remote_space 1000')
self.assertIn('quota unavailable', result.stdout)
self.assertTrue(all(p.name in self.remote_names() for p in [current]+copies))
def test_remote_quota_cancellation_is_propagated(self):
current, copies = self.pressure_copies()
self.quota_wrapper('cancel')
result = self.space_call('ensure_remote_space 1000', ok=False)
self.assertEqual(result.returncode, 143, result.stdout)
self.assertTrue(all(p.name in self.remote_names() for p in [current]+copies))
def test_dump_and_copy_failures_do_not_publish(self):
for tool in ['mariadb-dump','rsync','zstd']:
with self.subTest(tool=tool):