Recover disk space safely and complete metadata-only backup updates

tests/test_backup.py

@@ -198,6 +198,15 @@ p.write_text(json.dumps(s))
 printf '%s\\n' '{name}' >> "$TEST_EVENTS"
 if [[ ${{TEST_FAIL_TOOL:-}} == {name} ]]; then exit 42; fi
 if [[ {name} == rclone && ${{TEST_FAIL_CHECK:-}} == true && " $* " == *' check '* ]]; then exit 43; fi
+if [[ {name} == rclone && ${{TEST_METADATA_ONLY:-}} == true ]]; then
+  if [[ " $* " == *' --download '* || " $* " == *' cryptcheck '* ||
+        ( " $* " == *' check '* && " $* " != *' --size-only '* ) ||
+        ( " $* " == *' cat '* && " $* " == *'.tar.'* ) ||
+        " $* " == *' copy crypt:'* || " $* " == *' copyto crypt:'* ]]; then
+    echo forbidden-archive-read >> "$TEST_EVENTS"
+    exit 88
+  fi
+fi
 exec {real} "$@"
 '''
             path=self.bin/name
@@ -288,6 +297,8 @@ QUIESCE_SERVICES=(nc-cron)
         self.assertEqual(json.loads(self.state.read_text()),self.initial)
 
     def test_emergency_retention_reads_remote_and_preserves_minimum(self):
+        # Explicitly retain the old floor; default pressure policy is tested below.
+        with self.config.open('a') as f: f.write('SPACE_KEEP_LOCAL=3\n')
         self.crypt_config()
         current=self.backup()
         archives=current.parent
@@ -302,7 +313,8 @@ QUIESCE_SERVICES=(nc-cron)
             (path/'COMPLETE').write_text('APP_BACKUP_V2 '+HELPER.digest(path/'manifest.json')+'\n')
             subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'copy',str(path),'crypt:sets/'+name],check=True,capture_output=True)
             old.append(path)
-        # Full remote readback must fail for the oldest copy: retain it.
+        self.env['TEST_METADATA_ONLY']='true'
+        # Metadata check must fail for the oldest copy with a missing file: retain it.
         subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'deletefile','crypt:sets/'+old[0].name+'/wordpress.tar.zst'],check=True,capture_output=True)
         fake=self.bin/'df'
         fake.write_text('#!/bin/sh\nprintf "Filesystem 1B-blocks Used Available Use%% Mounted\\nroot 100 100 0 100%% /\\n"\n')
@@ -316,6 +328,106 @@ QUIESCE_SERVICES=(nc-cron)
         self.assertIn(old[1].name,remote)
         self.assertNotIn('systemctl stop',result.stdout)
 
+    def pressure_copies(self):
+        self.crypt_config()
+        current = self.backup()
+        copies = []
+        for day in range(1, 3):
+            path = current.parent / f'appbackup_2020-01-0{day}_00-00-00_aaaaaaaa'
+            shutil.copytree(current, path)
+            manifest = json.loads((path/'manifest.json').read_text())
+            manifest['run_id'] = path.name
+            (path/'manifest.json').write_text(json.dumps(manifest))
+            (path/'COMPLETE').write_text('APP_BACKUP_V2 '+HELPER.digest(path/'manifest.json')+'\n')
+            subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
+                            'copy', str(path), 'crypt:sets/'+path.name], check=True, capture_output=True)
+            copies.append(path)
+        self.env['TEST_METADATA_ONLY'] = 'true'
+        self.events.write_text('')
+        return current, copies
+
+    def fake_pressure(self, oldest, always=False):
+        # Model allocation pressure, then make reclaimed blocks visible to df.
+        fake = self.bin/'df'
+        fake.write_text('#!/usr/bin/env python3\nfrom pathlib import Path\n'
+                        f'available = 0 if {always!r} or Path({str(oldest)!r}).exists() else 10**12\n'
+                        'print("Filesystem 1B-blocks Used Available Use% Mounted")\n'
+                        'print(f"root 2000000000000 0 {available} 0% /")\n')
+        fake.chmod(0o700)
+
+    def test_pressure_can_break_three_copy_deadlock_and_stops_when_enough(self):
+        current, copies = self.pressure_copies()
+        self.fake_pressure(copies[0])
+        result = self.run_script('app-backup.sh', '--prepare-space')
+        self.assertIn('Space recovery complete', result.stdout)
+        self.assertFalse(copies[0].exists())
+        self.assertTrue(copies[1].exists())
+        self.assertTrue(current.exists())
+        remote = subprocess.check_output(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
+                                          'lsf', 'crypt:sets', '--dirs-only'], text=True)
+        self.assertIn(copies[0].name, remote)
+        self.assertNotIn('systemctl stop', self.events.read_text())
+
+    def test_pressure_keeps_last_uploaded_and_unconfirmed_copies(self):
+        current, copies = self.pressure_copies()
+        (copies[0]/'UPLOADED').unlink()
+        self.fake_pressure(copies[0], always=True)
+        result = self.run_script('app-backup.sh', '--prepare-space', ok=False)
+        self.assertTrue(current.exists())
+        self.assertTrue(copies[0].exists())
+        self.assertFalse(copies[1].exists())
+        self.assertIn('protected local minimum', result.stdout)
+        self.assertIn('protected unconfirmed', result.stdout)
+        self.assertIn('deficit ', result.stdout)
+        self.assertNotIn('systemctl stop', self.events.read_text())
+
+    def test_pressure_rejects_same_size_remote_manifest_and_marker_corruption(self):
+        current, copies = self.pressure_copies()
+        for path, file in zip(copies, ('manifest.json', 'COMPLETE')):
+            original = (path/file).read_bytes()
+            subprocess.run(['/usr/bin/rclone', '--config', str(self.base/'rclone.conf'),
+                            'rcat', 'crypt:sets/'+path.name+'/'+file],
+                           input=b'X'+original[1:], check=True, capture_output=True)
+        self.fake_pressure(copies[0], always=True)
+        result = self.run_script('app-backup.sh', '--prepare-space', ok=False)
+        self.assertTrue(all(p.exists() for p in [current]+copies))
+        self.assertIn('Remote identity check failed: manifest.json', result.stdout)
+        self.assertIn('Remote identity check failed: COMPLETE', result.stdout)
+
+    def test_pressure_cancellation_does_not_continue_to_next_copy(self):
+        current, copies = self.pressure_copies()
+        self.fake_pressure(copies[0], always=True)
+        wrapper = self.bin/'rclone'
+        wrapper.write_text('#!/bin/bash\nif [[ " $* " == *" check "* ]]; then exit 143; fi\nexec /usr/bin/rclone "$@"\n')
+        result = self.run_script('app-backup.sh', '--prepare-space', ok=False)
+        self.assertEqual(result.returncode, 143, result.stdout)
+        self.assertTrue(all(p.exists() for p in [current]+copies))
+        self.assertNotIn('before local removal: '+copies[1].name, result.stdout)
+
+    def test_pressure_preserves_all_copies_when_last_protected_copy_is_corrupt(self):
+        current, copies = self.pressure_copies()
+        (current/'wordpress.tar.zst').write_bytes(b'damaged')
+        self.fake_pressure(copies[0], always=True)
+        result = self.run_script('app-backup.sh', '--prepare-space', ok=False)
+        self.assertIn('Protected local copy is damaged', result.stdout)
+        self.assertTrue(all(p.exists() for p in [current]+copies))
+
+    def test_pressure_identity_cancellation_is_not_swallowed_by_comparison(self):
+        current, copies = self.pressure_copies()
+        self.fake_pressure(copies[0], always=True)
+        wrapper = self.bin/'rclone'
+        wrapper.write_text('#!/bin/bash\nif [[ " $* " == *" cat "* ]]; then exit 143; fi\nexec /usr/bin/rclone "$@"\n')
+        result = self.run_script('app-backup.sh', '--prepare-space', ok=False)
+        self.assertEqual(result.returncode, 143, result.stdout)
+        self.assertTrue(all(p.exists() for p in [current]+copies))
+
+    def test_pressure_floor_rejects_zero_and_unbounded_values(self):
+        for value in ('0', '-1', '01', '999999999999999999999', 'bad'):
+            with self.subTest(value=value):
+                with self.config.open('a') as f: f.write('SPACE_KEEP_LOCAL='+value+'\n')
+                result = self.run_script('app-backup.sh', '--check', ok=False)
+                self.assertIn('SPACE_KEEP_LOCAL must', result.stdout)
+
     def test_dump_and_copy_failures_do_not_publish(self):
         for tool in ['mariadb-dump','rsync','zstd']:
             with self.subTest(tool=tool):
@@ -655,26 +767,30 @@ else: raise SystemExit('Unexpected Docker command')
         docker.chmod(0o700)
         canonical=self.configure_upgrade()
         original=canonical.read_bytes()
+        self.env['TEST_METADATA_ONLY']='true'
         package=self.build_package()
         result=subprocess.run(['bash',str(package),'--foreground','--enable-restore-test'],env=self.env,capture_output=True,text=True,timeout=120)
         self.assertEqual(result.returncode,0,result.stdout+result.stderr)
         state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0])
         self.assertIn('SUCCESS',(state/'result.txt').read_text())
+        self.assertNotIn('forbidden-archive-read',self.events.read_text())
         message,body=self.read_report()
         self.assertEqual(str(message['From']),'app-backup@example.invalid')
         self.assertIn(' SUCCESS gemini.jr-it-services.de ',str(message['Subject']))
         self.assertIn('Host: gemini.jr-it-services.de',body.splitlines())
         self.assertIn('MAIL_HOSTNAME="gemini.jr-it-services.de"',canonical.read_text())
-        self.assertIn('Bereinigung: OK',body)
-        self.assertIn('[app-restore-test] SUCCESS',message['Subject'])
+        self.assertIn('Upload Status: OK (Dateien vorhanden',body)
+        self.assertNotIn('[app-restore-test]',message['Subject'])
+        self.assertFalse((self.base/'work/restore-test/last-success.json').exists())
+        self.assertNotIn('Remote-Restore-Pruefung erfolgreich', (state/'result.txt').read_text())
         saved=next((self.base/'logs').glob('report_*.eml'))
         backup_message=BytesParser(policy=policy.default).parsebytes(saved.read_bytes())
-        self.assertIn('OK (Inhalt und Abschlussmarker',backup_message.get_content())
+        self.assertIn('OK (Dateien vorhanden, Größen und Abschlussmarker',backup_message.get_content())
         self.assertEqual(Path('/usr/local/lib/app-backup/report.py').read_bytes(),(REPO/'lib/report.py').read_bytes())
         self.assertEqual(Path('/usr/local/lib/app-backup/restore_test.py').read_bytes(),(REPO/'lib/restore_test.py').read_bytes())
         self.assertTrue(Path('/etc/systemd/system/app-restore-test.timer').is_file())
         self.assertEqual(json.loads(self.state.read_text())['services']['app-restore-test.timer'],'active')
-        self.assertTrue((state/'verified-backup-run').is_file())
+        self.assertTrue((state/'uploaded-backup-run').is_file())
         self.assertIn('ENABLE_REMOTE_RETENTION=false',canonical.read_text())
         self.assertIn('RESTORE_TEST_MAX_WORK_GB=0',canonical.read_text())
         self.assertTrue(Path(str(self.events)+'.image-prepared').exists())
@@ -685,6 +801,47 @@ else: raise SystemExit('Unexpected Docker command')
         self.assertEqual(canonical.read_bytes(),original)
         self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'inactive')
 
+    def test_package_ai_rem_inclusion_is_idempotent_and_quiesces_jobs(self):
+        ai_paths=[Path('/srv/ai-rem'),Path('/etc/ai-rem')]
+        for path in ai_paths:
+            path.mkdir(parents=True,exist_ok=True)
+            (path/'snapshot.txt').write_text('ai-rem snapshot '+str(path))
+            self.addCleanup(shutil.rmtree,path)
+        units=['ai-rem-healthcheck.timer','ai-rem-backup.timer','ai-rem-prune.timer',
+               'ai-rem-healthcheck.service','ai-rem-backup.service','ai-rem-prune.service','ai-rem.service']
+        state=json.loads(self.state.read_text())
+        for unit in units: state['services'][unit]='active'
+        self.state.write_text(json.dumps(state))
+        canonical=self.configure_upgrade()
+        old_extra=self.src/'extra'
+        old_extra.mkdir()
+        (old_extra/'payload.txt').write_text('existing extra')
+        with canonical.open('a') as f:
+            f.write(f'EXTRA_PATHS=("{old_extra}")\n')
+        package=self.build_package()
+        self.env['TEST_METADATA_ONLY']='true'
+        for attempt in range(2):
+            self.events.write_text('')
+            result=subprocess.run(['bash',str(package),'--foreground','--enable-ai-rem'],
+                                  env=self.env,capture_output=True,text=True,timeout=120)
+            self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+            self.assertNotIn('forbidden-archive-read',self.events.read_text())
+            status=json.loads((self.base/'work/last-backup.json').read_text())
+            run=self.base/'work/archives'/status['run']
+            manifest=json.loads((run/'manifest.json').read_text())
+            extra=[a for a in manifest['archives'] if a['label'].startswith('extra-')]
+            self.assertEqual([(a['label'],a['source']) for a in extra],
+                             [('extra-0',str(old_extra)),('extra-1','/srv/ai-rem'),('extra-2','/etc/ai-rem')])
+            events=self.events.read_text()
+            for unit in units:
+                self.assertEqual(events.count('systemctl stop '+unit+' '),1)
+                self.assertLess(events.index('systemctl stop '+unit+' '),events.index('mariadb-dump'))
+                self.assertEqual(json.loads(self.state.read_text())['services'][unit],'active')
+            for unit in units[:-1]:
+                self.assertLess(events.index('systemctl stop '+unit+' '),events.index('systemctl stop ai-rem.service '))
+            self.assertLess(events.index('systemctl start ai-rem.service'),events.index('systemctl start ai-rem-healthcheck.timer'))
+            self.assertFalse((self.base/'work/restore-test/last-success.json').exists())
+
     def test_package_preflight_failure_preserves_live_configuration_and_timer(self):
         canonical=self.configure_upgrade()
         original=canonical.read_bytes()
@@ -832,7 +989,7 @@ else: raise SystemExit('Unexpected Docker command')
         self.assertIn('FAILED',(state/'result.txt').read_text())
         self.assertTrue((state/'rollback-update.sh').is_file())
         self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'inactive')
-        self.assertFalse((state/'verified-backup-run').exists())
+        self.assertFalse((state/'uploaded-backup-run').exists())
         rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30)
         self.assertEqual(rollback.returncode,0,rollback.stdout+rollback.stderr)
         self.assertEqual(canonical.read_bytes(),original)
@@ -1123,13 +1280,13 @@ REPORT_SECRET='config-content-must-not-enter-the-mail'
         self.assertIn('FAILED (exit 42)',body)
         self.assertEqual(json.loads(self.state.read_text()),self.initial)
 
-    def test_report_upload_readback_failure_is_not_reported_as_ok(self):
+    def test_report_upload_metadata_check_failure_is_not_reported_as_ok(self):
         self.enable_report()
         self.crypt_config()
         self.run_script('app-backup.sh',ok=False,TEST_FAIL_CHECK='true')
         _,body=self.read_report()
         self.assertIn('Status: FAIL',body)
-        self.assertIn('Upload Status: Nicht abgeschlossen: Rückleseprüfung',body)
+        self.assertIn('Upload Status: Nicht abgeschlossen: Dateiprüfung',body)
         self.assertNotIn('Upload Status: OK',body)
 
     def test_report_mta_failure_keeps_backup_result_and_local_message(self):
@@ -1162,7 +1319,7 @@ REPORT_SECRET='config-content-must-not-enter-the-mail'
         self.run_script('app-restore.sh','--local-run',run,'--verify-only')
         self.assertFalse((self.base/'mail.eml').exists())
 
-    def test_real_crypt_upload_readback_remote_restore(self):
+    def test_real_crypt_upload_metadata_check_remote_restore(self):
         self.crypt_config()
         # Exercise periodic stderr statistics during a real throttled crypt
         # transfer; they must not contaminate captured COMPLETE-marker stdout.
@@ -1175,10 +1332,10 @@ REPORT_SECRET='config-content-must-not-enter-the-mail'
         log=next((self.base/'logs').glob('backup_*.log')).read_text()
         self.assertIn('NOTICE:',log)
         self.assertIn('Remote upload started',log)
-        self.assertIn('Remote readback passed',log)
+        self.assertIn('Remote metadata check passed',log)
         self.run_script('app-restore.sh','--remote-run',run.name,'--verify-only')
 
-    def test_failed_readback_does_not_publish_remote_marker(self):
+    def test_failed_metadata_check_does_not_publish_remote_marker(self):
         self.crypt_config()
         result=self.run_script('app-backup.sh',ok=False,TEST_FAIL_CHECK='true')
         self.assertEqual(result.returncode,43)
@@ -1190,7 +1347,7 @@ REPORT_SECRET='config-content-must-not-enter-the-mail'
         proc=subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'cat',f'crypt:sets/{run.name}/COMPLETE'],capture_output=True)
         self.assertNotEqual(proc.returncode,0)
 
-    def inject_readback_failure(self, mode):
+    def inject_metadata_check_failure(self, mode):
         # The first check fails/stalls before real crypt I/O. Subsequent checks
         # use real rclone; event order proves COMPLETE was withheld until success.
         wrapper=self.bin/'rclone'
@@ -1199,9 +1356,9 @@ import os, pathlib, subprocess, sys, time
 args=sys.argv[1:]
 events=pathlib.Path(os.environ['TEST_EVENTS'])
 if 'check' in args:
- with events.open('a') as log: log.write('readback-check\\n')
- count=events.read_text().count('readback-check')
- mode=os.environ['TEST_READBACK_MODE']
+ with events.open('a') as log: log.write('metadata_check-check\\n')
+ count=events.read_text().count('metadata_check-check')
+ mode=os.environ['TEST_METADATA_CHECK_MODE']
  if mode=='signal': sys.exit(143)
  if mode=='hang' or (mode=='hang-once' and count==1): time.sleep(30)
  if mode=='fail-once' and count==1:
@@ -1212,23 +1369,23 @@ if 'copyto' in args:
 sys.exit(subprocess.call(['/usr/bin/rclone',*args]))
 ''')
         wrapper.chmod(0o755)
-        self.env['TEST_READBACK_MODE']=mode
+        self.env['TEST_METADATA_CHECK_MODE']=mode
 
-    def test_transient_readback_error_rechecks_before_publishing(self):
+    def test_transient_metadata_check_error_rechecks_before_publishing(self):
         self.crypt_config()
-        self.inject_readback_failure('fail-once')
+        self.inject_metadata_check_failure('fail-once')
         result=self.run_script('app-backup.sh')
         self.assertIn('attempt 2 passed',result.stdout)
         events=self.events.read_text()
-        self.assertEqual(events.count('readback-check'),2)
-        self.assertLess(events.rindex('readback-check'),events.index('publish-marker'))
+        self.assertEqual(events.count('metadata_check-check'),2)
+        self.assertLess(events.rindex('metadata_check-check'),events.index('publish-marker'))
         run=next((self.base/'work/archives').glob('appbackup_*'))
         self.assertTrue((run/'UPLOADED').exists())
         self.run_script('app-restore.sh','--remote-run',run.name,'--verify-only')
 
-    def test_stalled_readback_is_terminated_then_retried(self):
+    def test_stalled_metadata_check_is_terminated_then_retried(self):
         self.crypt_config()
-        self.inject_readback_failure('hang-once')
+        self.inject_metadata_check_failure('hang-once')
         with self.config.open('a') as f: f.write('RCLONE_CHECK_DEADLINE=2\n')
         result=self.run_script('app-backup.sh')
         self.assertIn('exit 124',result.stdout)
@@ -1236,24 +1393,24 @@ sys.exit(subprocess.call(['/usr/bin/rclone',*args]))
 
     def test_persistent_stall_is_bounded_and_preserves_local_backup(self):
         self.crypt_config()
-        self.inject_readback_failure('hang')
+        self.inject_metadata_check_failure('hang')
         with self.config.open('a') as f: f.write('RCLONE_CHECK_DEADLINE=1\nRCLONE_CHECK_ATTEMPTS=2\n')
         started=time.monotonic()
         result=self.run_script('app-backup.sh',ok=False)
         self.assertLess(time.monotonic()-started,20)
         self.assertEqual(result.returncode,124)
-        self.assertEqual(self.events.read_text().count('readback-check'),2)
+        self.assertEqual(self.events.read_text().count('metadata_check-check'),2)
         self.assert_remote_failure_retained(result)
 
-    def test_cancelled_readback_is_not_retried(self):
+    def test_cancelled_metadata_check_is_not_retried(self):
         self.crypt_config()
-        self.inject_readback_failure('signal')
+        self.inject_metadata_check_failure('signal')
         result=self.run_script('app-backup.sh',ok=False)
         self.assertEqual(result.returncode,143)
-        self.assertEqual(self.events.read_text().count('readback-check'),1)
+        self.assertEqual(self.events.read_text().count('metadata_check-check'),1)
         self.assert_remote_failure_retained(result)
 
-    def test_readback_limits_reject_unbounded_or_invalid_values(self):
+    def test_metadata_check_limits_reject_unbounded_or_invalid_values(self):
         for key,value in [('RCLONE_CHECK_ATTEMPTS','0'),('RCLONE_CHECK_ATTEMPTS','11'),
                           ('RCLONE_CHECK_DEADLINE','0'),('RCLONE_CHECK_DEADLINE','1h'),
                           ('RCLONE_CHECK_TIMEOUT','0'),('RCLONE_CHECK_LOW_LEVEL_RETRIES','0'),
@@ -1267,14 +1424,14 @@ sys.exit(subprocess.call(['/usr/bin/rclone',*args]))
                 finally:
                     self.config.write_text(original)
 
-    def test_invalid_crypt_header_stays_failed_after_retries(self):
+    def test_invalid_crypt_header_is_deferred_to_restore(self):
         self.crypt_config()
-        result=self.remote_failure('bad-header')
-        self.assertIn('bad magic string',result.stdout)
-        self.assertIn('attempt 3/3 failed',result.stdout)
-        self.assert_remote_failure_retained(result)
+        self.remote_failure('bad-header',ok=True)
+        run=next((self.base/'work/archives').glob('appbackup_*'))
+        self.assertTrue((run/'UPLOADED').exists())
+        self.run_script('app-restore.sh','--remote-run',run.name,'--verify-only',ok=False)
 
-    def remote_failure(self, mode, **env):
+    def remote_failure(self, mode, ok=False, **env):
         # Keep real crypt transfers. Inject errors at the command boundary, not
         # a pretend OneDrive server: provider retry behaviour is checked separately.
         wrapper = self.bin/'rclone'
@@ -1300,6 +1457,11 @@ if 'copy' in args:
  if mode=='upload':
   # Models an unsuccessful command even though the remote received bytes.
   sys.exit(44)
+ if mode in ('missing','wrong-size'):
+  config=args[args.index('--config')+1]
+  remote=args[args.index('copy')+2]+'/wordpress.tar.zst'
+  if mode=='missing': rc(['--config',config,'deletefile',remote],check=True)
+  else: rc(['--config',config,'rcat',remote],input=b'too short',check=True)
  if mode=='bad-header':
   # Damage the real encrypted header in place without changing object size.
   from pathlib import Path
@@ -1314,11 +1476,11 @@ if 'copy' in args:
   data=bytearray(rc([*prefix,'cat',remote],capture_output=True,check=True).stdout)
   data[len(data)//2]^=1
   # Same-size plaintext corruption, re-encrypted through the real crypt backend.
-  # The backup must read contents back, not accept a matching size alone.
+  # Daily metadata checks accept this; the restore checksum must reject it.
   rc([*prefix,'rcat',remote],input=bytes(data),check=True)
 ''')
         wrapper.chmod(0o755)
-        return self.run_script('app-backup.sh',ok=False,TEST_REMOTE_FAILURE=mode,**env)
+        return self.run_script('app-backup.sh',ok=ok,TEST_REMOTE_FAILURE=mode,**env)
 
     def assert_remote_failure_retained(self, result, marker_present=False):
         status=json.loads((self.base/'work/last-backup.json').read_text())
@@ -1363,13 +1525,22 @@ if 'copy' in args:
             self.assertTrue(path.is_dir())
             self.assertIn(path.name,remote)
 
-    def test_same_size_remote_corruption_blocks_completion(self):
+    def test_same_size_remote_corruption_is_deferred_to_restore(self):
         self.crypt_config()
-        result=self.remote_failure('corrupt')
-        self.assert_remote_failure_retained(result)
-        self.assertFalse((self.base/'work/last-successful-backup.json').exists())
+        self.remote_failure('corrupt',ok=True)
+        run=next((self.base/'work/archives').glob('appbackup_*'))
+        self.assertTrue((run/'UPLOADED').exists())
+        self.run_script('app-restore.sh','--remote-run',run.name,'--verify-only',ok=False)
 
-    def test_environment_cannot_weaken_remote_readback(self):
+    def test_missing_remote_archive_blocks_completion(self):
+        self.crypt_config()
+        self.assert_remote_failure_retained(self.remote_failure('missing'))
+
+    def test_wrong_size_remote_archive_blocks_completion(self):
+        self.crypt_config()
+        self.assert_remote_failure_retained(self.remote_failure('wrong-size'))
+
+    def test_environment_cannot_weaken_remote_metadata_check(self):
         self.crypt_config()
         result=self.remote_failure('corrupt',RCLONE_SIZE_ONLY='true')
         self.assertIn('Unsupported rclone environment override: RCLONE_SIZE_ONLY',result.stdout)
@@ -1397,7 +1568,7 @@ if 'copy' in args:
     def test_marker_read_mismatch_blocks_upload_success(self):
         self.crypt_config()
         result=self.remote_failure('marker-read')
-        # Archives passed readback and COMPLETE was written, but its final
+        # Archives passed metadata_check and COMPLETE was written, but its final
         # confirmation failed. Local success and retention must remain blocked.
         self.assert_remote_failure_retained(result,marker_present=True)
         self.assertIn('Remote completion marker mismatch',result.stdout)