Retry restore-test downloads safely and expose bounded diagnostics

tests/test_restore_test.py

@@ -12,6 +12,7 @@ import sys
 import tarfile
 import tempfile
 import time
+from types import SimpleNamespace
 import unittest
 from email.parser import BytesParser
 from email import policy
@@ -54,6 +55,7 @@ RCLONE_CONFIG="{self.crypt}"
 RCLONE_REMOTE_BASE="crypt:sets"
 RCLONE_RETRIES=1
 RCLONE_LOW_LEVEL_RETRIES=1
+RCLONE_CHECK_RETRY_DELAY=0
 RESTORE_TEST_MIN_FREE_GB=0
 RESTORE_TEST_MAX_WORK_GB=1
 RESTORE_TEST_TIMEOUT=30
@@ -154,6 +156,131 @@ MAIL_SENDMAIL_BIN="{self.sendmail}"
         self.assertEqual(self.state()['status'],'FAIL')
         self.clean()
 
+    def download_failure(self, mode, target='wordpress.tar.gz'):
+        # Real crypt transfers surround injected stream failures. stderr includes
+        # a fake credential to prove that diagnostics never copy raw messages.
+        wrapper = self.base/'test-rclone'
+        counter = self.base/'attempts'
+        wrapper.write_text(f'''#!/usr/bin/env python3
+import os, pathlib, subprocess, sys, time
+args = sys.argv[1:]
+if 'cat' in args and any(a.endswith('/' + {target!r}) for a in args):
+ counter = pathlib.Path({str(counter)!r})
+ attempt = int(counter.read_text()) + 1 if counter.exists() else 1
+ counter.write_text(str(attempt))
+ assert args[args.index('--timeout')+1] == '300s', args
+ assert args[args.index('--low-level-retries')+1] == '3', args
+ mode = {mode!r}
+ if mode == 'signal': sys.exit(143)
+ if mode == 'oversize':
+  sys.stdout.buffer.write(b'x'*1048576)
+  sys.exit(0)
+ if mode == 'hang' or (mode == 'hang-once' and attempt == 1):
+  sys.stdout.buffer.write(b'partial')
+  sys.stdout.buffer.flush()
+  time.sleep(20)
+ if mode in ('always-fail', 'stderr-flood') or (mode == 'fail-once' and attempt == 1):
+  # Larger than the pipe capacity; the supervisor must drain stderr concurrently.
+  sys.stderr.write(('https://example.invalid/?token=SECRET-DO-NOT-LOG HTTP 503 bad magic string\\n') * (2000 if mode == 'stderr-flood' else 1))
+  sys.stdout.buffer.write(b'partial')
+  sys.exit(5)
+os.execv('/usr/bin/rclone', ['/usr/bin/rclone', *args])
+''')
+        wrapper.chmod(0o700)
+        with self.config.open('a') as f: f.write(f'RCLONE_BIN="{wrapper}"\n')
+        return counter
+
+    def test_partial_download_restarts_cleanly_and_checks_real_content(self):
+        counter = self.download_failure('fail-once')
+        output = self.execute()
+        self.assertEqual(counter.read_text(),'2')
+        self.assertIn('crypt-header',output)
+        self.assertIn('http-503',output)
+        self.assertIn('exit 5',output)
+        self.assertNotIn('SECRET-DO-NOT-LOG',output)
+        self.assertEqual(self.state()['status'],'SUCCESS')
+        self.assertTrue(all(c['phase']=='complete' for c in self.state()['components']))
+        self.clean()
+
+    def test_partial_metadata_read_is_not_appended_on_retry(self):
+        counter = self.download_failure('fail-once', 'manifest.json')
+        self.execute()
+        self.assertEqual(counter.read_text(),'2')
+        self.assertEqual(self.state()['status'],'SUCCESS')
+        self.clean()
+
+    def test_database_stderr_stays_private_with_operation_and_exit_code(self):
+        rehearsal = restore_test.Rehearsal(SimpleNamespace(workdir=str(self.work), mode='run', timeout=10))
+        with self.assertRaises(restore_test.CommandError) as result:
+            rehearsal.command([sys.executable, '-c', "import sys; sys.stderr.write('SQL-SECRET'); sys.exit(2)"],
+                              operation='Database import')
+        self.assertIn('Database import failed (exit 2', str(result.exception))
+        self.assertNotIn('SQL-SECRET', str(result.exception))
+        self.assertNotIn('SQL-SECRET', (self.root/'restore-test.log').read_text())
+        self.clean()
+
+    def test_persistent_download_error_reports_phase_and_preserves_previous_success(self):
+        self.execute()
+        previous = (self.root/'last-success.json').read_bytes()
+        counter = self.download_failure('always-fail')
+        output = self.execute(ok=False)
+        self.assertEqual(counter.read_text(),'3')
+        self.assertIn('Component wordpress / download: Remote cat failed (exit 5',output)
+        self.assertIn('crypt-header',self.state()['error'])
+        self.assertEqual((self.root/'last-success.json').read_bytes(),previous)
+        for path in ['state.json','restore-test.log','report.eml']:
+            self.assertNotIn('SECRET-DO-NOT-LOG',(self.root/path).read_text())
+        self.clean()
+
+    def test_remote_stderr_flood_does_not_deadlock_or_leak(self):
+        counter = self.download_failure('stderr-flood')
+        output = self.execute(ok=False)
+        self.assertEqual(counter.read_text(),'3')
+        self.assertIn('http-503',output)
+        self.assertNotIn('SECRET-DO-NOT-LOG',output)
+        self.assertLess(len(output),12000)
+        self.clean()
+
+    def test_stalled_download_retries_from_empty_file(self):
+        counter = self.download_failure('hang-once')
+        with self.config.open('a') as f: f.write('RCLONE_CHECK_DEADLINE=2\n')
+        output = self.execute()
+        self.assertIn('deadline exceeded',output)
+        self.assertEqual(counter.read_text(),'2')
+        self.clean()
+
+    def test_stalled_download_exhausts_attempts_and_cleans(self):
+        counter = self.download_failure('hang')
+        with self.config.open('a') as f: f.write('RCLONE_CHECK_DEADLINE=2\nRCLONE_CHECK_ATTEMPTS=2\n')
+        self.execute(ok=False)
+        self.assertEqual(counter.read_text(),'2')
+        self.assertIn('deadline exceeded',self.state()['error'])
+        self.assertEqual(self.state()['cleanup'],'OK')
+        self.clean()
+
+    def test_download_signal_does_not_retry(self):
+        counter = self.download_failure('signal')
+        self.execute(ok=False)
+        self.assertEqual(counter.read_text(),'1')
+        self.assertIn('exit 143',self.state()['error'])
+        self.clean()
+
+    def test_oversized_download_does_not_retry(self):
+        counter = self.download_failure('oversize')
+        self.execute(ok=False)
+        self.assertEqual(counter.read_text(),'1')
+        self.assertIn('exceeds expected size',self.state()['error'])
+        self.clean()
+
+    def test_global_timeout_stops_download_retry_backoff(self):
+        counter = self.download_failure('always-fail')
+        with self.config.open('a') as f:
+            f.write('RESTORE_TEST_TIMEOUT=5\nRCLONE_CHECK_RETRY_DELAY=10\n')
+        self.execute(ok=False)
+        self.assertEqual(counter.read_text(),'1')
+        self.assertIn('time limit exceeded',self.state()['error'])
+        self.clean()
+
     def test_disk_budget_refuses_work_and_cleans(self):
         with self.config.open('a') as f: f.write('RESTORE_TEST_MAX_WORK_GB=0.000001\n')
         self.execute(ok=False)