Prepare MariaDB restore tests and recover backup disk space safely

lib/restore_test.py

@@ -137,6 +137,19 @@ class Rehearsal:
         if time.monotonic() >= self.deadline:
             raise TestError('Restore-test time limit exceeded')
         free = shutil.disk_usage(self.root).free
+        if (force and self.args.mode == 'run' and self.args.remote_check and
+                free < self.args.min_free_gb * GIB + additional):
+            # Only between subprocesses: do not overwrite the active-child journal.
+            self.log.info('Attempting safe local space recovery before next component step')
+            self.monitor = False
+            try:
+                result = self.remote('space', str(int(additional)), capture=1024*1024)
+                self.log.info('%s', result.decode(errors='replace').strip())
+            except TestError as exc:
+                raise TestError('Free-space reserve unavailable after safe recovery; protected data retained') from exc
+            finally:
+                self.monitor = True
+            free = shutil.disk_usage(self.root).free
         if free < self.args.min_free_gb * GIB + additional:
             raise TestError('Free-space reserve reached; test stopped and scratch will be cleaned')
         if force or time.monotonic() >= self.next_usage:
@@ -145,7 +158,7 @@ class Rehearsal:
                 for name in directories + files:
                     self.used += (Path(path) / name).lstat().st_blocks * 512
             self.next_usage = time.monotonic() + 2
-        if self.used + additional > self.args.max_work_gb * GIB:
+        if self.args.max_work_gb and self.used + additional > self.args.max_work_gb * GIB:
             raise TestError('Restore-test disk budget exceeded')
 
     def command(self, command, *, capture=0, output=None, limit=None, stdin=None, timeout=120):
@@ -459,7 +472,7 @@ def io_bytes(raw):
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('mode', choices=('run', 'check', 'cleanup'))
+    parser.add_argument('mode', choices=('run', 'check', 'cleanup', 'prepare'))
     for key in ('workdir', 'prefix', 'remote', 'run', 'image', 'hostname', 'to', 'sender', 'sendmail', 'mail'):
         parser.add_argument('--' + key, required=True)
     for key in ('remote-check', 'config'):
@@ -470,12 +483,35 @@ def main():
     for key in ('max-work-gb', 'min-free-gb'):
         parser.add_argument('--' + key, type=float, required=True)
     args = parser.parse_args()
-    if not (0 < args.max_work_gb <= 100000 and 0 <= args.min_free_gb <= 100000 and
+    if not (0 <= args.max_work_gb <= 100000 and 0 <= args.min_free_gb <= 100000 and
             1 <= args.max_age_hours <= 8760 and 1 <= args.timeout <= 86400 and 1 <= args.mail_timeout <= 300):
         parser.error('Invalid restore-test limits')
     if os.geteuid() != 0:
         parser.error('Run as root via app-restore-test.sh')
     test = Rehearsal(args)
+    if args.mode == 'prepare':
+        def stop_prepare(signum, _frame):
+            raise TestError('Installer probe interrupted: ' + str(signum))
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            signal.signal(sig, stop_prepare)
+        # Installer smoke test, separate from monthly backup acceptance/status.
+        # Refuse an interrupted rehearsal until its regular cleanup reported it.
+        if any((test.root / name).exists() for name in ('scratch', 'db-created', 'child.json')):
+            raise TestError('Run restore-test --cleanup before DB preparation')
+        try:
+            test.preflight()
+            test.scratch.mkdir(mode=0o700)
+            test.monitor = True
+            sql = test.scratch / 'probe.sql'
+            sql.write_text('CREATE TABLE installer_probe (id INT PRIMARY KEY); INSERT INTO installer_probe VALUES (1);\n')
+            test.disk(additional=512*1024*1024, force=True)
+            if test.database(sql) != 1:
+                raise TestError('Installer database probe returned unexpected table count')
+        finally:
+            test.monitor = False
+            test.cleanup()
+        test.log.info('Installer DB initialization, import, check and cleanup passed')
+        return 0
     if args.mode == 'cleanup' and not test.root.exists():
         return 0
     if args.mode == 'check':