Prepare MariaDB restore tests and recover backup disk space safely

tests/test_backup.py

@@ -272,6 +272,49 @@ QUIESCE_SERVICES=(nc-cron)
         self.assertEqual((self.wp/'payload.txt').read_text(),'payload for web')
         self.assertEqual(json.loads(self.state.read_text()),self.initial)
 
+    def test_failed_backup_scratch_is_removed_after_service_recovery(self):
+        result=self.run_script('app-backup.sh',ok=False,TEST_FAIL_TOOL='zstd')
+        self.assertFalse(list((self.base/'work/staging').iterdir()))
+        self.assertFalse(list((self.base/'work/archives').iterdir()))
+        self.assertEqual(json.loads(self.state.read_text()),self.initial)
+        self.assertIn('Retained work: none',result.stdout)
+
+    def test_space_failure_happens_before_writer_shutdown(self):
+        with self.config.open('a') as f: f.write('MIN_FREE_GB=100000\n')
+        result=self.run_script('app-backup.sh',ok=False)
+        self.assertIn('Insufficient space after safe cleanup',result.stdout)
+        self.assertNotIn('systemctl stop',self.events.read_text())
+        self.assertEqual(json.loads(self.state.read_text()),self.initial)
+
+    def test_emergency_retention_reads_remote_and_preserves_minimum(self):
+        self.crypt_config()
+        current=self.backup()
+        archives=current.parent
+        old=[]
+        for day in range(1,5):
+            name=f'appbackup_2020-01-0{day}_00-00-00_aaaaaaaa'
+            path=archives/name
+            shutil.copytree(current,path)
+            manifest=json.loads((path/'manifest.json').read_text())
+            manifest['run_id']=name
+            (path/'manifest.json').write_text(json.dumps(manifest))
+            (path/'COMPLETE').write_text('APP_BACKUP_V2 '+HELPER.digest(path/'manifest.json')+'\n')
+            subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'copy',str(path),'crypt:sets/'+name],check=True,capture_output=True)
+            old.append(path)
+        # Full remote readback must fail for the oldest copy: retain it.
+        subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'deletefile','crypt:sets/'+old[0].name+'/wordpress.tar.zst'],check=True,capture_output=True)
+        fake=self.bin/'df'
+        fake.write_text('#!/bin/sh\nprintf "Filesystem 1B-blocks Used Available Use%% Mounted\\nroot 100 100 0 100%% /\\n"\n')
+        fake.chmod(0o700)
+        result=self.run_script('app-backup.sh',ok=False)
+        self.assertIn('verification failed',result.stdout)
+        self.assertTrue(old[0].exists())
+        self.assertFalse(old[1].exists())
+        self.assertTrue(all(p.exists() for p in old[2:]+[current]))
+        remote=subprocess.check_output(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'lsf','crypt:sets','--dirs-only'],text=True)
+        self.assertIn(old[1].name,remote)
+        self.assertNotIn('systemctl stop',result.stdout)
+
     def test_dump_and_copy_failures_do_not_publish(self):
         for tool in ['mariadb-dump','rsync','zstd']:
             with self.subTest(tool=tool):
@@ -345,7 +388,7 @@ if '--exclude=/*' in sys.argv:
         self.state.write_text(json.dumps(state))
         wrapper=self.bin/'rsync'
         wrapper.write_text('''#!/usr/bin/env bash
-if [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* ]]; then
+if [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* && " $* " != *" --dry-run "* ]]; then
  systemctl start httpd
  systemctl start php-fpm
  systemctl start gitea
@@ -366,7 +409,7 @@ import json,os,subprocess,sys
 from pathlib import Path
 result=subprocess.run(['/usr/bin/rsync',*sys.argv[1:]])
 if result.returncode: sys.exit(result.returncode)
-if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
+if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv and '--dry-run' not in sys.argv:
  p=Path(os.environ['TEST_STATE'])
  state=json.loads(p.read_text())
  state['services']['httpd']='''+repr(final_state)+'''
@@ -461,7 +504,7 @@ if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
         wrapper=self.bin/'rsync'
         # Signal only during the real copy, after services have stopped. The
         # new access probes run earlier and must not trigger this recovery test.
-        wrapper.write_text('#!/usr/bin/env bash\nif [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* ]]; then\n echo waiting > "$TEST_EVENTS.waiting"\n sleep 2\nfi\nexec /usr/bin/rsync "$@"\n')
+        wrapper.write_text('#!/usr/bin/env bash\nif [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* && " $* " != *" --dry-run "* ]]; then\n echo waiting > "$TEST_EVENTS.waiting"\n sleep 2\nfi\nexec /usr/bin/rsync "$@"\n')
         proc=subprocess.Popen(['bash',str(REPO/'usr-local-sbin/app-backup.sh'),'--config',str(self.config)],env=self.env,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
         try:
             for _ in range(200):
@@ -587,19 +630,31 @@ if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
         self.enable_report()
         with self.config.open('a') as f:
             f.write('MAIL_HOSTNAME="gemini.jr-it-services.de"\nENABLE_RESTORE_TEST=true\nRESTORE_TEST_DB_IMAGE="test-image"\nRESTORE_TEST_MIN_FREE_GB=0\n')
-        # The real isolated DB import is covered separately. Updater preflight
-        # only inspects the prepared local image; it must never pull one.
+        # Real image preparation/import is also exercised in the opt-in Docker suite.
         docker=self.bin/'docker'
         docker.write_text('''#!/usr/bin/env python3
-import json,sys
-assert sys.argv[1:] == ['--host=unix:///var/run/docker.sock','image','inspect','test-image']
-print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql':{}}}}]))
+import json,sys,os
+from pathlib import Path
+prepared=Path(os.environ["TEST_EVENTS"]+".image-prepared")
+args=sys.argv[2:]
+if args[0]=='info': print('/tmp')
+elif args[0]=='pull':
+ prepared.write_text('image downloaded once'); print('prepared')
+elif args[:2]==['image','inspect']:
+ if not prepared.exists(): raise SystemExit(1)
+ if '--format' in args: print('sha256:test-image')
+ else: print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql':{}}}}]))
+elif args[0]=='run' and '--rm' in args: print('mariadbd Ver 11.8.6-MariaDB')
+elif args[0]=='run': print('test-container')
+elif args[0]=='exec': print('1')
+elif args[:2]==['container','ls']: pass
+else: raise SystemExit('Unexpected Docker command')
 ''')
         docker.chmod(0o700)
         canonical=self.configure_upgrade()
         original=canonical.read_bytes()
         package=self.build_package()
-        result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=120)
+        result=subprocess.run(['bash',str(package),'--foreground','--enable-restore-test'],env=self.env,capture_output=True,text=True,timeout=120)
         self.assertEqual(result.returncode,0,result.stdout+result.stderr)
         state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0])
         self.assertIn('SUCCESS',(state/'result.txt').read_text())
@@ -608,13 +663,19 @@ print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql
         self.assertIn(' SUCCESS gemini.jr-it-services.de ',str(message['Subject']))
         self.assertIn('Host: gemini.jr-it-services.de',body.splitlines())
         self.assertIn('MAIL_HOSTNAME="gemini.jr-it-services.de"',canonical.read_text())
-        self.assertIn('OK (Inhalt und Abschlussmarker',body)
+        self.assertIn('Bereinigung: OK',body)
+        self.assertIn('[app-restore-test] SUCCESS',message['Subject'])
+        saved=next((self.base/'logs').glob('report_*.eml'))
+        backup_message=BytesParser(policy=policy.default).parsebytes(saved.read_bytes())
+        self.assertIn('OK (Inhalt und Abschlussmarker',backup_message.get_content())
         self.assertEqual(Path('/usr/local/lib/app-backup/report.py').read_bytes(),(REPO/'lib/report.py').read_bytes())
         self.assertEqual(Path('/usr/local/lib/app-backup/restore_test.py').read_bytes(),(REPO/'lib/restore_test.py').read_bytes())
         self.assertTrue(Path('/etc/systemd/system/app-restore-test.timer').is_file())
         self.assertEqual(json.loads(self.state.read_text())['services']['app-restore-test.timer'],'active')
         self.assertTrue((state/'verified-backup-run').is_file())
         self.assertIn('ENABLE_REMOTE_RETENTION=false',canonical.read_text())
+        self.assertIn('RESTORE_TEST_MAX_WORK_GB=0',canonical.read_text())
+        self.assertTrue(Path(str(self.events)+'.image-prepared').exists())
         self.assertEqual(self.cnf.stat().st_mode & 0o777,0o600)
         self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active')
         rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30)
@@ -827,7 +888,7 @@ if '--no-block' not in args:
         self.systemd_launcher()
         wrapper=self.bin/tool
         if tool=='rsync':
-            match='[[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* ]]'
+            match='[[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* && " $* " != *" --dry-run "* ]]'
         else:
             match='[[ " $* " == *" copy "* ]]'
         wrapper.write_text('#!/usr/bin/env bash\nif '+match+'; then\n echo waiting > "$TEST_EVENTS.waiting"\n sleep 120\nfi\nexec /usr/bin/'+tool+' "$@"\n')

tests/test_restore_test.py

@@ -311,6 +311,83 @@ exec /usr/bin/rclone "$@"
         with self.config.open('a') as f:
             f.write(f'ENABLE_DB_DUMPS=true\nWP_DB_NAME=production\nWP_DB_CNF=/unused\nRESTORE_TEST_DB_IMAGE="{image}"\nRESTORE_TEST_TIMEOUT=120\n')
 
+    @unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
+    def test_installer_detects_mariadb_and_probes_real_image(self):
+        self.prepare_database(b'CREATE TABLE example (id INT);')
+        cnf=self.base/'backup.cnf'
+        cnf.write_text('[client]\n')
+        cnf.chmod(0o600)
+        client=self.base/'version-client'
+        client.write_text('#!/bin/sh\necho 10.11.18-MariaDB\n')
+        client.chmod(0o700)
+        with self.config.open('a') as f:
+            f.write(f'WP_DB_CNF="{cnf}"\nDB_CLIENT_BIN="{client}"\nRESTORE_TEST_DB_IMAGE=""\n')
+        result=subprocess.run(['bash',str(REPO/'lib/prepare-restore-test.sh'),str(self.config)],capture_output=True,text=True,timeout=120)
+        self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+        self.assertIn('RESTORE_TEST_DB_IMAGE=mariadb:10.11',self.config.read_text())
+        self.assertIn('initialization, import, check and cleanup passed',result.stdout)
+        self.assertFalse((self.root/'last-success.json').exists())
+        self.no_container()
+
+    @unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
+    def test_installer_rejects_wrong_mariadb_series_without_config_change(self):
+        self.prepare_database(b'CREATE TABLE example (id INT);')
+        cnf=self.base/'backup.cnf'
+        cnf.write_text('[client]\n')
+        cnf.chmod(0o600)
+        client=self.base/'version-client'
+        client.write_text('#!/bin/sh\necho 11.4.1-MariaDB\n')
+        client.chmod(0o700)
+        with self.config.open('a') as f: f.write(f'WP_DB_CNF="{cnf}"\nDB_CLIENT_BIN="{client}"\n')
+        original=self.config.read_bytes()
+        result=subprocess.run(['bash',str(REPO/'lib/prepare-restore-test.sh'),str(self.config)],capture_output=True,text=True,timeout=60)
+        self.assertNotEqual(result.returncode,0,result.stdout+result.stderr)
+        self.assertIn('does not match production MariaDB series',result.stdout+result.stderr)
+        self.assertEqual(self.config.read_bytes(),original)
+        self.assertFalse(self.root.exists())
+
+    def test_space_recovery_uses_parent_lock_and_preserves_three_local_sets(self):
+        self.work.chmod(0o700)
+        for name in ('archives','staging','restore'): (self.work/name).mkdir(mode=0o700)
+        old=[]
+        for day in range(1,5):
+            run=f'appbackup_2020-01-0{day}_00-00-00_aaaaaaaa'
+            path=self.work/'archives'/run
+            shutil.copytree(self.folder,path)
+            path.chmod(0o700)
+            data=json.loads((path/'manifest.json').read_text())
+            data['run_id']=run
+            (path/'manifest.json').write_text(json.dumps(data))
+            (path/'COMPLETE').write_text('APP_BACKUP_V2 '+backup_set.digest(path/'manifest.json')+'\n')
+            (path/'UPLOADED').write_text('verified previously\n')
+            subprocess.run(['rclone','--config',str(self.crypt),'copy',str(path),'crypt:sets/'+run],check=True,capture_output=True)
+            old.append(path)
+        with self.config.open('a') as f: f.write('RESTORE_TEST_MIN_FREE_GB=0.001\n')
+        # Simulate pressure consistently for the Python supervisor and Bash df.
+        # Once one old local set is removed, the real free-space figure returns.
+        bindir=self.base/'bin'
+        bindir.mkdir()
+        script=self.base/'supervisor.py'
+        script.write_text('import os,runpy,shutil,sys\nfrom pathlib import Path\noriginal=shutil.disk_usage\npressure=Path('+repr(str(old[0]))+')\ndef usage(p):\n value=original(p)\n return value._replace(free=0) if pressure.exists() else value\nshutil.disk_usage=usage\npath=sys.argv.pop(1)\nsys.path.insert(0,str(Path(path).parent))\nrunpy.run_path(path,run_name="__main__")\n')
+        python=bindir/'python3'
+        python.write_text('#!/bin/sh\ncase "$1" in */restore_test.py) exec /usr/bin/python3 "'+str(script)+'" "$@";; esac\nexec /usr/bin/python3 "$@"\n')
+        python.chmod(0o700)
+        df=bindir/'df'
+        df.write_text('#!/bin/sh\nif test -d "'+str(old[0])+'"; then printf "Filesystem 1B-blocks Used Available Use%% Mounted\\nroot 100 100 0 100%% /\\n"; else exec /usr/bin/df "$@"; fi\n')
+        df.chmod(0o700)
+        result=subprocess.run(['bash',str(REPO/'usr-local-sbin/app-restore-test.sh'),'--config',str(self.config)],env=dict(os.environ,PATH=str(bindir)+':'+os.environ['PATH']),capture_output=True,text=True,timeout=60)
+        self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+        self.assertFalse(old[0].exists())
+        self.assertTrue(all(p.exists() for p in old[1:]))
+        self.assertEqual(self.state()['status'],'SUCCESS')
+        self.clean()
+
+    def test_adaptive_budget_accepts_zero(self):
+        with self.config.open('a') as f: f.write('RESTORE_TEST_MAX_WORK_GB=0\n')
+        self.execute()
+        self.clean()
+        self.assertEqual(self.state()['status'],'SUCCESS')
+
     def no_container(self):
         self.clean()
         names=subprocess.check_output(restore_test.DOCKER+['ps','-a','--filter',f'label={restore_test.OWNER_LABEL}', '--format','{{.Names}}'],text=True)

tests/test_space.py

@@ -0,0 +1,81 @@
+"""Recovery policy tests: reclaim only owned scratch, protect recovery/last copies."""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'lib'))
+import space
+
+
+@unittest.skipUnless(os.geteuid() == 0, 'root-only cleanup policy')
+class SpaceRecovery(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.work = Path(self.temp.name)
+        self.staging = self.work/'staging'
+        self.archives = self.work/'archives'
+        self.restore = self.work/'restore'
+        for p in (self.staging, self.archives, self.restore): p.mkdir(mode=0o700)
+        self.run = 'appbackup_2026-09-12_17-07-56_bd1f6193'
+        self.stage = self.staging/(self.run+'.5ICERN')
+        self.stage.mkdir(mode=0o700)
+        (self.stage/'meta').mkdir()
+        (self.stage/'meta/run-id.txt').write_text(self.run+'\n')
+        (self.stage/'meta/format.txt').write_text('format=2\ncompressor=zstd\n')
+        self.partial = self.archives/('.partial-'+self.run)
+        self.partial.mkdir(mode=0o700)
+        (self.partial/'meta.tar.zst').write_text('incomplete scratch')
+
+    def clean(self):
+        space.clean(self.work, self.staging, self.archives, 'appbackup', self.restore)
+
+    def test_gemini_failed_compression_reclaims_both_trees(self):
+        unknown = self.staging/'keep-manual-files'
+        unknown.mkdir(mode=0o700)
+        self.clean()
+        self.assertFalse(self.stage.exists())
+        self.assertFalse(self.partial.exists())
+        self.assertTrue(unknown.exists())
+
+    def test_recovery_journal_blocks_all_cleanup(self):
+        (self.work/('recovery-'+self.run+'.json')).write_text('{}')
+        with self.assertRaisesRegex(ValueError, 'recovery journal'): self.clean()
+        self.assertTrue(self.stage.exists())
+        self.assertTrue(self.partial.exists())
+
+    def test_complete_partial_is_preserved(self):
+        (self.partial/'COMPLETE').write_text('possibly only complete snapshot')
+        self.clean()
+        self.assertTrue(self.stage.exists())
+        self.assertTrue(self.partial.exists())
+
+    def test_nested_symlink_does_not_delete_target(self):
+        target = self.work/'production'
+        target.mkdir()
+        (target/'data').write_text('live')
+        (self.stage/'link').symlink_to(target)
+        self.clean()
+        self.assertEqual((target/'data').read_text(), 'live')
+
+    def test_mount_is_never_removed(self):
+        mount = self.stage/'mounted'
+        mount.mkdir()
+        with patch('space.os.path.ismount', side_effect=lambda p: p == mount):
+            with self.assertRaisesRegex(ValueError, 'mounted'): self.clean()
+        self.assertTrue(mount.exists())
+
+    def test_verification_download_recovered_manual_restore_kept(self):
+        verify = self.restore/(self.run+'.abcdef')
+        verify.mkdir(mode=0o700)
+        (verify/'.verify-only').write_text(self.run+'\n')
+        manual = self.restore/(self.run+'.ghijkl')
+        manual.mkdir(mode=0o700)
+        self.clean()
+        self.assertFalse(verify.exists())
+        self.assertTrue(manual.exists())