Prepare MariaDB restore tests and recover backup disk space safely

tests/test_backup.py

@@ -272,6 +272,49 @@ QUIESCE_SERVICES=(nc-cron)
         self.assertEqual((self.wp/'payload.txt').read_text(),'payload for web')
         self.assertEqual(json.loads(self.state.read_text()),self.initial)
 
+    def test_failed_backup_scratch_is_removed_after_service_recovery(self):
+        result=self.run_script('app-backup.sh',ok=False,TEST_FAIL_TOOL='zstd')
+        self.assertFalse(list((self.base/'work/staging').iterdir()))
+        self.assertFalse(list((self.base/'work/archives').iterdir()))
+        self.assertEqual(json.loads(self.state.read_text()),self.initial)
+        self.assertIn('Retained work: none',result.stdout)
+
+    def test_space_failure_happens_before_writer_shutdown(self):
+        with self.config.open('a') as f: f.write('MIN_FREE_GB=100000\n')
+        result=self.run_script('app-backup.sh',ok=False)
+        self.assertIn('Insufficient space after safe cleanup',result.stdout)
+        self.assertNotIn('systemctl stop',self.events.read_text())
+        self.assertEqual(json.loads(self.state.read_text()),self.initial)
+
+    def test_emergency_retention_reads_remote_and_preserves_minimum(self):
+        self.crypt_config()
+        current=self.backup()
+        archives=current.parent
+        old=[]
+        for day in range(1,5):
+            name=f'appbackup_2020-01-0{day}_00-00-00_aaaaaaaa'
+            path=archives/name
+            shutil.copytree(current,path)
+            manifest=json.loads((path/'manifest.json').read_text())
+            manifest['run_id']=name
+            (path/'manifest.json').write_text(json.dumps(manifest))
+            (path/'COMPLETE').write_text('APP_BACKUP_V2 '+HELPER.digest(path/'manifest.json')+'\n')
+            subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'copy',str(path),'crypt:sets/'+name],check=True,capture_output=True)
+            old.append(path)
+        # Full remote readback must fail for the oldest copy: retain it.
+        subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'deletefile','crypt:sets/'+old[0].name+'/wordpress.tar.zst'],check=True,capture_output=True)
+        fake=self.bin/'df'
+        fake.write_text('#!/bin/sh\nprintf "Filesystem 1B-blocks Used Available Use%% Mounted\\nroot 100 100 0 100%% /\\n"\n')
+        fake.chmod(0o700)
+        result=self.run_script('app-backup.sh',ok=False)
+        self.assertIn('verification failed',result.stdout)
+        self.assertTrue(old[0].exists())
+        self.assertFalse(old[1].exists())
+        self.assertTrue(all(p.exists() for p in old[2:]+[current]))
+        remote=subprocess.check_output(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'lsf','crypt:sets','--dirs-only'],text=True)
+        self.assertIn(old[1].name,remote)
+        self.assertNotIn('systemctl stop',result.stdout)
+
     def test_dump_and_copy_failures_do_not_publish(self):
         for tool in ['mariadb-dump','rsync','zstd']:
             with self.subTest(tool=tool):
@@ -345,7 +388,7 @@ if '--exclude=/*' in sys.argv:
         self.state.write_text(json.dumps(state))
         wrapper=self.bin/'rsync'
         wrapper.write_text('''#!/usr/bin/env bash
-if [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* ]]; then
+if [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* && " $* " != *" --dry-run "* ]]; then
  systemctl start httpd
  systemctl start php-fpm
  systemctl start gitea
@@ -366,7 +409,7 @@ import json,os,subprocess,sys
 from pathlib import Path
 result=subprocess.run(['/usr/bin/rsync',*sys.argv[1:]])
 if result.returncode: sys.exit(result.returncode)
-if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
+if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv and '--dry-run' not in sys.argv:
  p=Path(os.environ['TEST_STATE'])
  state=json.loads(p.read_text())
  state['services']['httpd']='''+repr(final_state)+'''
@@ -461,7 +504,7 @@ if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
         wrapper=self.bin/'rsync'
         # Signal only during the real copy, after services have stopped. The
         # new access probes run earlier and must not trigger this recovery test.
-        wrapper.write_text('#!/usr/bin/env bash\nif [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* ]]; then\n echo waiting > "$TEST_EVENTS.waiting"\n sleep 2\nfi\nexec /usr/bin/rsync "$@"\n')
+        wrapper.write_text('#!/usr/bin/env bash\nif [[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* && " $* " != *" --dry-run "* ]]; then\n echo waiting > "$TEST_EVENTS.waiting"\n sleep 2\nfi\nexec /usr/bin/rsync "$@"\n')
         proc=subprocess.Popen(['bash',str(REPO/'usr-local-sbin/app-backup.sh'),'--config',str(self.config)],env=self.env,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
         try:
             for _ in range(200):
@@ -587,19 +630,31 @@ if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
         self.enable_report()
         with self.config.open('a') as f:
             f.write('MAIL_HOSTNAME="gemini.jr-it-services.de"\nENABLE_RESTORE_TEST=true\nRESTORE_TEST_DB_IMAGE="test-image"\nRESTORE_TEST_MIN_FREE_GB=0\n')
-        # The real isolated DB import is covered separately. Updater preflight
-        # only inspects the prepared local image; it must never pull one.
+        # Real image preparation/import is also exercised in the opt-in Docker suite.
         docker=self.bin/'docker'
         docker.write_text('''#!/usr/bin/env python3
-import json,sys
-assert sys.argv[1:] == ['--host=unix:///var/run/docker.sock','image','inspect','test-image']
-print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql':{}}}}]))
+import json,sys,os
+from pathlib import Path
+prepared=Path(os.environ["TEST_EVENTS"]+".image-prepared")
+args=sys.argv[2:]
+if args[0]=='info': print('/tmp')
+elif args[0]=='pull':
+ prepared.write_text('image downloaded once'); print('prepared')
+elif args[:2]==['image','inspect']:
+ if not prepared.exists(): raise SystemExit(1)
+ if '--format' in args: print('sha256:test-image')
+ else: print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql':{}}}}]))
+elif args[0]=='run' and '--rm' in args: print('mariadbd Ver 11.8.6-MariaDB')
+elif args[0]=='run': print('test-container')
+elif args[0]=='exec': print('1')
+elif args[:2]==['container','ls']: pass
+else: raise SystemExit('Unexpected Docker command')
 ''')
         docker.chmod(0o700)
         canonical=self.configure_upgrade()
         original=canonical.read_bytes()
         package=self.build_package()
-        result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=120)
+        result=subprocess.run(['bash',str(package),'--foreground','--enable-restore-test'],env=self.env,capture_output=True,text=True,timeout=120)
         self.assertEqual(result.returncode,0,result.stdout+result.stderr)
         state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0])
         self.assertIn('SUCCESS',(state/'result.txt').read_text())
@@ -608,13 +663,19 @@ print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql
         self.assertIn(' SUCCESS gemini.jr-it-services.de ',str(message['Subject']))
         self.assertIn('Host: gemini.jr-it-services.de',body.splitlines())
         self.assertIn('MAIL_HOSTNAME="gemini.jr-it-services.de"',canonical.read_text())
-        self.assertIn('OK (Inhalt und Abschlussmarker',body)
+        self.assertIn('Bereinigung: OK',body)
+        self.assertIn('[app-restore-test] SUCCESS',message['Subject'])
+        saved=next((self.base/'logs').glob('report_*.eml'))
+        backup_message=BytesParser(policy=policy.default).parsebytes(saved.read_bytes())
+        self.assertIn('OK (Inhalt und Abschlussmarker',backup_message.get_content())
         self.assertEqual(Path('/usr/local/lib/app-backup/report.py').read_bytes(),(REPO/'lib/report.py').read_bytes())
         self.assertEqual(Path('/usr/local/lib/app-backup/restore_test.py').read_bytes(),(REPO/'lib/restore_test.py').read_bytes())
         self.assertTrue(Path('/etc/systemd/system/app-restore-test.timer').is_file())
         self.assertEqual(json.loads(self.state.read_text())['services']['app-restore-test.timer'],'active')
         self.assertTrue((state/'verified-backup-run').is_file())
         self.assertIn('ENABLE_REMOTE_RETENTION=false',canonical.read_text())
+        self.assertIn('RESTORE_TEST_MAX_WORK_GB=0',canonical.read_text())
+        self.assertTrue(Path(str(self.events)+'.image-prepared').exists())
         self.assertEqual(self.cnf.stat().st_mode & 0o777,0o600)
         self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active')
         rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30)
@@ -827,7 +888,7 @@ if '--no-block' not in args:
         self.systemd_launcher()
         wrapper=self.bin/tool
         if tool=='rsync':
-            match='[[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* ]]'
+            match='[[ " $* " != *" --list-only "* && " $* " != *" --exclude=/* "* && " $* " != *" --dry-run "* ]]'
         else:
             match='[[ " $* " == *" copy "* ]]'
         wrapper.write_text('#!/usr/bin/env bash\nif '+match+'; then\n echo waiting > "$TEST_EVENTS.waiting"\n sleep 120\nfi\nexec /usr/bin/'+tool+' "$@"\n')