Harden remote verification against environment overrides and marker normalization

tests/test_backup.py

@@ -1,5 +1,6 @@
 """Integration tests use real tar/rsync/rclone crypt and a real isolated MariaDB.
-Only systemd/occ are simulated: production services never receive test commands.
+systemd/occ are simulated; error paths inject failures around real commands.
+Production services never receive test commands.
 """
 import gzip
 import hashlib
@@ -971,6 +972,133 @@ if '--no-block' not in args:
         proc=subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),'cat',f'crypt:sets/{run.name}/COMPLETE'],capture_output=True)
         self.assertNotEqual(proc.returncode,0)
 
+    def remote_failure(self, mode, **env):
+        # Keep real crypt transfers. Inject errors at the command boundary, not
+        # a pretend OneDrive server: provider retry behaviour is checked separately.
+        wrapper = self.bin/'rclone'
+        wrapper.write_text('''#!/usr/bin/env python3
+import os, subprocess, sys
+args=sys.argv[1:]
+mode=os.environ.get('TEST_REMOTE_FAILURE','')
+def rc(arguments, **kw):
+ return subprocess.run(['/usr/bin/rclone',*arguments], **kw)
+if 'copyto' in args and mode=='marker-write':
+ sys.exit(44)
+if 'cat' in args and mode=='marker-read':
+ print('incorrect completion marker')
+ sys.exit(0)
+if 'cat' in args and mode=='marker-newline':
+ data=rc(args,capture_output=True,check=True).stdout
+ sys.stdout.buffer.write(data+b'\\n')
+ sys.exit(0)
+result=rc(args)
+if result.returncode:
+ sys.exit(result.returncode)
+if 'copy' in args:
+ if mode=='upload':
+  # Models an unsuccessful command even though the remote received bytes.
+  sys.exit(44)
+ if mode=='corrupt':
+  config=args[args.index('--config')+1]
+  remote=args[args.index('copy')+2]+'/wordpress.tar.zst'
+  prefix=['--config',config]
+  data=bytearray(rc([*prefix,'cat',remote],capture_output=True,check=True).stdout)
+  data[len(data)//2]^=1
+  # Same-size plaintext corruption, re-encrypted through the real crypt backend.
+  # The backup must read contents back, not accept a matching size alone.
+  rc([*prefix,'rcat',remote],input=bytes(data),check=True)
+''')
+        wrapper.chmod(0o755)
+        return self.run_script('app-backup.sh',ok=False,TEST_REMOTE_FAILURE=mode,**env)
+
+    def assert_remote_failure_retained(self, result, marker_present=False):
+        status=json.loads((self.base/'work/last-backup.json').read_text())
+        self.assertNotEqual(status['exit_code'],0)
+        run=self.base/'work/archives'/status['run']
+        HELPER.load_set(run)  # Local hashes/manifest must still be valid.
+        self.assertFalse((run/'UPLOADED').exists())
+        self.assertEqual(json.loads(self.state.read_text()),self.initial)
+        self.assertFalse(list(self.runtime.glob('app-backup-guard-*')))
+        self.assertNotIn('-- Local retention:',result.stdout)
+        self.assertNotIn('-- Remote retention:',result.stdout)
+        marker=subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),
+                               'cat',f'crypt:sets/{run.name}/COMPLETE'],capture_output=True)
+        self.assertEqual(marker.returncode==0,marker_present)
+        return run
+
+    def test_upload_failure_preserves_previous_success_and_retention_candidates(self):
+        self.crypt_config()
+        current=self.backup()
+        previous=(self.base/'work/last-successful-backup.json').read_bytes()
+        old=[]
+        for day in range(1,4):
+            name=f'appbackup_2020-01-0{day}_00-00-00_aaaaaaaa'
+            path=current.parent/name
+            shutil.copytree(current,path)
+            manifest=json.loads((path/'manifest.json').read_text())
+            manifest['run_id']=name
+            (path/'manifest.json').write_text(json.dumps(manifest))
+            (path/'COMPLETE').write_text('APP_BACKUP_V2 '+HELPER.digest(path/'manifest.json')+'\n')
+            subprocess.run(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),
+                            'copy',str(path),'crypt:sets/'+name],check=True,capture_output=True)
+            old.append(path)
+        with self.config.open('a') as f:
+            f.write('ENABLE_REMOTE_RETENTION=true\nRETENTION_KEEP_LAST=2\nLOCAL_RETENTION_DAYS=1\nREMOTE_RETENTION_DAYS=1\n')
+        result=self.remote_failure('upload')
+        self.assert_remote_failure_retained(result)
+        self.assertEqual(result.returncode,44)
+        self.assertEqual((self.base/'work/last-successful-backup.json').read_bytes(),previous)
+        remote=subprocess.check_output(['/usr/bin/rclone','--config',str(self.base/'rclone.conf'),
+                                        'lsf','crypt:sets','--dirs-only'],text=True)
+        for path in old:
+            self.assertTrue(path.is_dir())
+            self.assertIn(path.name,remote)
+
+    def test_same_size_remote_corruption_blocks_completion(self):
+        self.crypt_config()
+        result=self.remote_failure('corrupt')
+        self.assert_remote_failure_retained(result)
+        self.assertFalse((self.base/'work/last-successful-backup.json').exists())
+
+    def test_environment_cannot_weaken_remote_readback(self):
+        self.crypt_config()
+        result=self.remote_failure('corrupt',RCLONE_SIZE_ONLY='true')
+        self.assertIn('Unsupported rclone environment override: RCLONE_SIZE_ONLY',result.stdout)
+        self.assertNotIn('systemctl stop',self.events.read_text())
+
+    def test_unsupported_rclone_environment_rejected_before_remote_access(self):
+        self.crypt_config()
+        for key,value in [('RCLONE_EXCLUDE','*'),('RCLONE_DRY_RUN','true'),
+                          ('RCLONE_IGNORE_ERRORS','true'),('RCLONE_NO_CHECK_CERTIFICATE','true')]:
+            with self.subTest(key=key):
+                self.events.write_text('')
+                result=self.run_script('app-backup.sh','--check',ok=False,**{key:value})
+                self.assertIn('Unsupported rclone environment override: '+key,result.stdout)
+                self.assertNotIn('rclone',self.events.read_text())
+                self.assertNotIn('systemctl stop',self.events.read_text())
+        # Supported settings remain usable when an administrator exports them.
+        self.run_script('app-backup.sh','--check',RCLONE_RETRIES='2',RCLONE_STATS_INTERVAL='1m')
+
+    def test_marker_write_failure_blocks_upload_success(self):
+        self.crypt_config()
+        result=self.remote_failure('marker-write')
+        self.assert_remote_failure_retained(result)
+        self.assertEqual(result.returncode,44)
+
+    def test_marker_read_mismatch_blocks_upload_success(self):
+        self.crypt_config()
+        result=self.remote_failure('marker-read')
+        # Archives passed readback and COMPLETE was written, but its final
+        # confirmation failed. Local success and retention must remain blocked.
+        self.assert_remote_failure_retained(result,marker_present=True)
+        self.assertIn('Remote completion marker mismatch',result.stdout)
+
+    def test_marker_extra_newline_blocks_upload_success(self):
+        self.crypt_config()
+        result=self.remote_failure('marker-newline')
+        self.assert_remote_failure_retained(result,marker_present=True)
+        self.assertIn('Remote completion marker mismatch',result.stdout)
+
     def test_unsafe_crypt_and_source_overlap_rejected(self):
         path=self.crypt_config()
         with path.open('a') as f: f.write('no_data_encryption = true\n')