Bound and retry complete remote readback checks

tests/test_backup.py

@@ -217,6 +217,7 @@ COMPRESSOR=zstd
 ENABLE_UPLOAD=false
 ENABLE_MAIL_REPORT=false
 ENABLE_REMOTE_RETENTION=false
+RCLONE_CHECK_RETRY_DELAY=0
 ENABLE_WORDPRESS=true
 WP_DIR='{self.wp}'
 ENABLE_NEXTCLOUD=true
@@ -1178,12 +1179,100 @@ REPORT_SECRET='config-content-must-not-enter-the-mail'
 
     def test_failed_readback_does_not_publish_remote_marker(self):
         self.crypt_config()
-        self.run_script('app-backup.sh',ok=False,TEST_FAIL_CHECK='true')
+        result=self.run_script('app-backup.sh',ok=False,TEST_FAIL_CHECK='true')
+        self.assertEqual(result.returncode,43)
+        self.assertIn('attempt 3/3 failed',result.stdout)
         run=next((self.base/'work/archives').glob('appbackup_*'))
+        self.assertIn('Retained local backup: '+str(run),result.stdout)
+        HELPER.load_set(run)
         self.assertFalse((run/'UPLOADED').exists())
         proc=subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'cat',f'crypt:sets/{run.name}/COMPLETE'],capture_output=True)
         self.assertNotEqual(proc.returncode,0)
 
+    def inject_readback_failure(self, mode):
+        # The first check fails/stalls before real crypt I/O. Subsequent checks
+        # use real rclone; event order proves COMPLETE was withheld until success.
+        wrapper=self.bin/'rclone'
+        wrapper.write_text('''#!/usr/bin/env python3
+import os, pathlib, subprocess, sys, time
+args=sys.argv[1:]
+events=pathlib.Path(os.environ['TEST_EVENTS'])
+if 'check' in args:
+ with events.open('a') as log: log.write('readback-check\\n')
+ count=events.read_text().count('readback-check')
+ mode=os.environ['TEST_READBACK_MODE']
+ if mode=='signal': sys.exit(143)
+ if mode=='hang' or (mode=='hang-once' and count==1): time.sleep(30)
+ if mode=='fail-once' and count==1:
+  print('not an encrypted file - bad magic string',file=sys.stderr)
+  sys.exit(1)
+if 'copyto' in args:
+ with events.open('a') as log: log.write('publish-marker\\n')
+sys.exit(subprocess.call(['/usr/bin/rclone',*args]))
+''')
+        wrapper.chmod(0o755)
+        self.env['TEST_READBACK_MODE']=mode
+
+    def test_transient_readback_error_rechecks_before_publishing(self):
+        self.crypt_config()
+        self.inject_readback_failure('fail-once')
+        result=self.run_script('app-backup.sh')
+        self.assertIn('attempt 2 passed',result.stdout)
+        events=self.events.read_text()
+        self.assertEqual(events.count('readback-check'),2)
+        self.assertLess(events.rindex('readback-check'),events.index('publish-marker'))
+        run=next((self.base/'work/archives').glob('appbackup_*'))
+        self.assertTrue((run/'UPLOADED').exists())
+        self.run_script('app-restore.sh','--remote-run',run.name,'--verify-only')
+
+    def test_stalled_readback_is_terminated_then_retried(self):
+        self.crypt_config()
+        self.inject_readback_failure('hang-once')
+        with self.config.open('a') as f: f.write('RCLONE_CHECK_DEADLINE=2\n')
+        result=self.run_script('app-backup.sh')
+        self.assertIn('exit 124',result.stdout)
+        self.assertIn('attempt 2 passed',result.stdout)
+
+    def test_persistent_stall_is_bounded_and_preserves_local_backup(self):
+        self.crypt_config()
+        self.inject_readback_failure('hang')
+        with self.config.open('a') as f: f.write('RCLONE_CHECK_DEADLINE=1\nRCLONE_CHECK_ATTEMPTS=2\n')
+        started=time.monotonic()
+        result=self.run_script('app-backup.sh',ok=False)
+        self.assertLess(time.monotonic()-started,20)
+        self.assertEqual(result.returncode,124)
+        self.assertEqual(self.events.read_text().count('readback-check'),2)
+        self.assert_remote_failure_retained(result)
+
+    def test_cancelled_readback_is_not_retried(self):
+        self.crypt_config()
+        self.inject_readback_failure('signal')
+        result=self.run_script('app-backup.sh',ok=False)
+        self.assertEqual(result.returncode,143)
+        self.assertEqual(self.events.read_text().count('readback-check'),1)
+        self.assert_remote_failure_retained(result)
+
+    def test_readback_limits_reject_unbounded_or_invalid_values(self):
+        for key,value in [('RCLONE_CHECK_ATTEMPTS','0'),('RCLONE_CHECK_ATTEMPTS','11'),
+                          ('RCLONE_CHECK_DEADLINE','0'),('RCLONE_CHECK_DEADLINE','1h'),
+                          ('RCLONE_CHECK_TIMEOUT','0'),('RCLONE_CHECK_LOW_LEVEL_RETRIES','0'),
+                          ('RCLONE_CHECK_RETRY_DELAY','-1')]:
+            with self.subTest(key=key,value=value):
+                original=self.config.read_text()
+                self.config.write_text(original+f'\n{key}={value}\n')
+                try:
+                    self.run_script('app-backup.sh','--check',ok=False)
+                    self.assertNotIn('systemctl stop',self.events.read_text())
+                finally:
+                    self.config.write_text(original)
+
+    def test_invalid_crypt_header_stays_failed_after_retries(self):
+        self.crypt_config()
+        result=self.remote_failure('bad-header')
+        self.assertIn('bad magic string',result.stdout)
+        self.assertIn('attempt 3/3 failed',result.stdout)
+        self.assert_remote_failure_retained(result)
+
     def remote_failure(self, mode, **env):
         # Keep real crypt transfers. Inject errors at the command boundary, not
         # a pretend OneDrive server: provider retry behaviour is checked separately.
@@ -1210,6 +1299,13 @@ if 'copy' in args:
  if mode=='upload':
   # Models an unsuccessful command even though the remote received bytes.
   sys.exit(44)
+ if mode=='bad-header':
+  # Damage the real encrypted header in place without changing object size.
+  from pathlib import Path
+  root=Path(os.environ['TEST_EVENTS']).parent/'encrypted'
+  for path in root.rglob('*'):
+   if path.is_file():
+    with path.open('r+b') as f: f.write(b'BADMAGIC')
  if mode=='corrupt':
   config=args[args.index('--config')+1]
   remote=args[args.index('copy')+2]+'/wordpress.tar.zst'