528 lines
25 KiB
Python
528 lines
25 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only offsite restore rehearsal. Only marked scratch data may be deleted.
|
|
|
|
The shell launcher holds the shared backup lock through this process. Docker has
|
|
one bind mount, no network/ports, no restart policy and no persistent log driver.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
from email.message import EmailMessage
|
|
from email.policy import SMTP
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from logging.handlers import RotatingFileHandler
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import selectors
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
|
|
import backup_set
|
|
|
|
GIB = 1024 ** 3
|
|
MARKER = 'app-backup-restore-test-v1\n'
|
|
DOCKER = ['docker', '--host=unix:///var/run/docker.sock']
|
|
OWNER_LABEL = 'de.jr-it-services.app-backup.restore-test'
|
|
|
|
|
|
class TestError(Exception):
|
|
pass
|
|
|
|
|
|
def atomic_json(path, data):
|
|
temp = path.with_suffix('.tmp')
|
|
temp.write_text(json.dumps(data, ensure_ascii=True, indent=2) + '\n')
|
|
temp.replace(path)
|
|
|
|
|
|
def private_root(workdir, create=True):
|
|
work = Path(workdir)
|
|
# Do not follow aliases to source paths or accept broad filesystem roots.
|
|
if (not work.is_absolute() or work.resolve() != work or len(work.parts) < 3 or
|
|
any(c in str(work) for c in ':\n\r')):
|
|
raise TestError('Unsafe WORKDIR for restore test')
|
|
if work.exists() and (work.stat().st_uid != 0 or work.stat().st_mode & 0o022):
|
|
raise TestError('WORKDIR must be root-owned and not writable by group/others')
|
|
root = work / 'restore-test'
|
|
if root.is_symlink():
|
|
raise TestError('Restore-test root must not be a symlink')
|
|
if not root.exists():
|
|
if not create:
|
|
return root
|
|
root.mkdir(mode=0o700, parents=True)
|
|
(root / '.owner').write_text(MARKER)
|
|
marker = root / '.owner'
|
|
if (root.stat().st_uid != 0 or root.stat().st_mode & 0o077 or
|
|
not marker.is_file() or marker.is_symlink() or marker.read_text() != MARKER):
|
|
raise TestError('Unrecognized/non-private restore-test directory; refusing cleanup')
|
|
# These paths are written by us only. Never follow replacements into live data.
|
|
for name in ('.owner', 'scratch', 'state.json', 'state.tmp', 'last-success.json',
|
|
'last-success.tmp', 'report.eml', 'restore-test.log', 'restore-test.log.1', 'db-created',
|
|
'child.json', 'child.tmp'):
|
|
if (root / name).is_symlink():
|
|
raise TestError('Symlink inside restore-test control directory')
|
|
return root
|
|
|
|
|
|
class Rehearsal:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.root = private_root(args.workdir, create=args.mode != 'cleanup')
|
|
self.scratch = self.root / 'scratch'
|
|
self.owner = hashlib.sha256(str(self.root).encode()).hexdigest()[:16]
|
|
self.container = 'app-backup-restore-test-' + self.owner
|
|
self.started = time.time()
|
|
self.deadline = time.monotonic() + args.timeout
|
|
self.state = dict(status='RUNNING', started_epoch=int(self.started), run='',
|
|
components=[], cleanup='PENDING', error='')
|
|
self.log = logging.getLogger('restore-test')
|
|
self.log.setLevel(logging.INFO)
|
|
for handler in self.log.handlers:
|
|
handler.close()
|
|
self.log.handlers.clear()
|
|
if self.root.exists():
|
|
handler = RotatingFileHandler(self.root / 'restore-test.log', maxBytes=512*1024, backupCount=1)
|
|
handler.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
|
|
self.log.addHandler(handler)
|
|
self.log.addHandler(logging.StreamHandler(sys.stdout))
|
|
self.next_usage = 0
|
|
self.used = 0
|
|
self.monitor = False
|
|
|
|
def remember_child(self, proc):
|
|
stat = Path(f'/proc/{proc.pid}/stat').read_text().rsplit(')', 1)[1].split()
|
|
atomic_json(self.root / 'child.json', dict(pid=proc.pid, start=stat[19]))
|
|
|
|
def stop_orphan(self):
|
|
marker = self.root / 'child.json'
|
|
if not marker.exists():
|
|
return
|
|
child = json.loads(marker.read_text())
|
|
pid = child['pid']
|
|
if not isinstance(pid, int) or pid <= 1:
|
|
raise TestError('Invalid child-process marker')
|
|
def alive():
|
|
try:
|
|
stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
|
|
return stat[19] == child['start'] and stat[0] != 'Z' and os.getpgid(pid) == pid
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return False
|
|
if alive():
|
|
os.killpg(pid, signal.SIGTERM)
|
|
end = time.monotonic() + 5
|
|
while alive() and time.monotonic() < end:
|
|
time.sleep(0.1)
|
|
if alive():
|
|
os.killpg(pid, signal.SIGKILL)
|
|
end = time.monotonic() + 5
|
|
while alive() and time.monotonic() < end:
|
|
time.sleep(0.1)
|
|
if alive():
|
|
raise TestError('Child process could not be stopped; cleanup deferred')
|
|
marker.unlink()
|
|
|
|
def save(self):
|
|
atomic_json(self.root / 'state.json', self.state)
|
|
|
|
def disk(self, additional=0, force=False):
|
|
if not self.monitor:
|
|
return
|
|
if time.monotonic() >= self.deadline:
|
|
raise TestError('Restore-test time limit exceeded')
|
|
free = shutil.disk_usage(self.root).free
|
|
if free < self.args.min_free_gb * GIB + additional:
|
|
raise TestError('Free-space reserve reached; test stopped and scratch will be cleaned')
|
|
if force or time.monotonic() >= self.next_usage:
|
|
self.used = 0
|
|
for path, directories, files in os.walk(self.scratch, followlinks=False):
|
|
for name in directories + files:
|
|
self.used += (Path(path) / name).lstat().st_blocks * 512
|
|
self.next_usage = time.monotonic() + 2
|
|
if self.used + additional > self.args.max_work_gb * GIB:
|
|
raise TestError('Restore-test disk budget exceeded')
|
|
|
|
def command(self, command, *, capture=0, output=None, limit=None, stdin=None, timeout=120):
|
|
"""Bound stdout and runtime, monitor disk, reap children even on interruption.
|
|
|
|
stderr is intentionally not copied: DB errors can contain restored data.
|
|
Errors identify the failed operation without logging SQL or credentials.
|
|
"""
|
|
result = bytearray()
|
|
total = 0
|
|
end = time.monotonic() + timeout
|
|
proc = subprocess.Popen(command, stdin=stdin or subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE if capture or output is not None else subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL, start_new_session=True)
|
|
selector = selectors.DefaultSelector()
|
|
if proc.stdout:
|
|
selector.register(proc.stdout, selectors.EVENT_READ)
|
|
try:
|
|
self.remember_child(proc)
|
|
while selector.get_map() or proc.poll() is None:
|
|
self.disk()
|
|
if time.monotonic() >= end:
|
|
raise TestError('Timed out: ' + Path(command[0]).name)
|
|
if not selector.get_map():
|
|
time.sleep(0.1)
|
|
continue
|
|
for key, _ in selector.select(0.2):
|
|
chunk = os.read(key.fd, 64*1024)
|
|
if not chunk:
|
|
selector.unregister(key.fileobj)
|
|
continue
|
|
total += len(chunk)
|
|
if total > (limit if limit is not None else capture):
|
|
raise TestError('Remote output exceeds expected size')
|
|
if output is not None:
|
|
self.disk(additional=len(chunk))
|
|
output.write(chunk)
|
|
else:
|
|
result.extend(chunk)
|
|
if proc.wait() != 0:
|
|
raise TestError('Command failed: ' + Path(command[0]).name)
|
|
return bytes(result)
|
|
finally:
|
|
selector.close()
|
|
if proc.poll() is None:
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
proc.wait()
|
|
if proc.stdout:
|
|
proc.stdout.close()
|
|
(self.root / 'child.json').unlink(missing_ok=True)
|
|
|
|
def remote(self, *args, **kwargs):
|
|
return self.command(['bash', self.args.remote_check, self.args.config, *args],
|
|
timeout=self.args.timeout, **kwargs)
|
|
|
|
def cleanup(self):
|
|
self.stop_orphan()
|
|
# db-created is written BEFORE docker run. A lost CLI reply cannot orphan a DB.
|
|
if (self.root / 'db-created').exists():
|
|
names = self.command(DOCKER + ['container', 'ls', '-a', '--filter',
|
|
f'label={OWNER_LABEL}={self.owner}', '--format', '{{.Names}}'],
|
|
capture=4096).decode().splitlines()
|
|
if any(name != self.container for name in names):
|
|
raise TestError('Unexpected container ownership; refusing cleanup')
|
|
if names:
|
|
self.command(DOCKER + ['container', 'rm', '-f', '-v', self.container])
|
|
(self.root / 'db-created').unlink()
|
|
if self.scratch.exists():
|
|
# shutil.rmtree uses fd-relative traversal on Linux; nested symlinks
|
|
# are unlinked, never traversed. No broad cleanup of WORKDIR or Docker.
|
|
if self.scratch.is_symlink() or self.scratch.is_mount():
|
|
raise TestError('Unsafe scratch directory; refusing cleanup')
|
|
shutil.rmtree(self.scratch)
|
|
self.state['cleanup'] = 'OK'
|
|
|
|
def preflight(self):
|
|
self.monitor = True
|
|
for executable in ('tar', 'gzip', 'zstd'):
|
|
if not shutil.which(executable):
|
|
raise TestError('Missing tool: ' + executable)
|
|
if any(label.startswith('db-') for label in self.args.expected):
|
|
self.check_image()
|
|
self.disk(force=True)
|
|
|
|
def check_image(self):
|
|
if hasattr(self, 'image_id'):
|
|
return
|
|
if not self.args.image or self.args.image.startswith('-') or any(c.isspace() for c in self.args.image):
|
|
raise TestError('Set RESTORE_TEST_DB_IMAGE to a locally prepared MariaDB image matching production')
|
|
raw = self.command(DOCKER + ['image', 'inspect', self.args.image], capture=128*1024)
|
|
info = json.loads(raw)[0]
|
|
volumes = info.get('Config', {}).get('Volumes') or {}
|
|
if set(volumes) - {'/var/lib/mysql'}:
|
|
raise TestError('Test image declares unexpected volumes')
|
|
# Store immutable local image ID: a tag change during a test cannot switch binaries.
|
|
self.image_id = info['Id']
|
|
|
|
def select(self):
|
|
base = self.args.remote.rstrip('/')
|
|
if self.args.run:
|
|
runs = [self.args.run]
|
|
else:
|
|
listing = self.remote('lsf', base, '--dirs-only', '--max-depth', '1', capture=1024*1024).decode()
|
|
runs = sorted((line.rstrip('/') for line in listing.splitlines()
|
|
if backup_set.RUN.fullmatch(line.rstrip('/')) and
|
|
line.startswith(self.args.prefix + '_')), reverse=True)
|
|
for run in runs:
|
|
if not backup_set.RUN.fullmatch(run) or not run.startswith(self.args.prefix + '_'):
|
|
raise TestError('Invalid remote run name')
|
|
files = self.remote('lsf', base + '/' + run, '--files-only', '--max-depth', '1', capture=1024*1024).decode().splitlines()
|
|
if 'COMPLETE' not in files:
|
|
if self.args.run:
|
|
raise TestError('Selected backup is incomplete')
|
|
self.log.info('Incomplete backup skipped: %s', run)
|
|
continue
|
|
self.state['run'] = run
|
|
self.save()
|
|
for name, maximum in [('manifest.json', 1024*1024), ('COMPLETE', 256)]:
|
|
data = self.remote('cat', base + '/' + run + '/' + name, capture=maximum)
|
|
(self.scratch / name).write_bytes(data)
|
|
manifest = backup_set.load_set(self.scratch, metadata_only=True)
|
|
if manifest['run_id'] != run:
|
|
raise TestError('Remote directory and manifest run differ')
|
|
age = time.time() - manifest['created_epoch']
|
|
self.state['backup_age_hours'] = round(age / 3600, 2)
|
|
if age < -300 or age > self.args.max_age_hours * 3600:
|
|
raise TestError('Backup is stale or dated in the future')
|
|
labels = {item['label'] for item in manifest['archives']}
|
|
if set(self.args.expected) - labels:
|
|
raise TestError('Backup is missing configured components')
|
|
allowed = {item['file'] for item in manifest['archives']} | {'manifest.json', 'COMPLETE', 'UPLOADED'}
|
|
if set(files) != allowed and set(files) != allowed - {'UPLOADED'}:
|
|
raise TestError('Remote backup has missing or unexpected files')
|
|
return manifest
|
|
raise TestError('No completed remote backup found')
|
|
|
|
def expand_size(self, archive):
|
|
# Full tar validation is performed first by the shared validator. A second
|
|
# streaming pass estimates logical size, including sparse files and metadata.
|
|
unpacked = 0
|
|
count = 0
|
|
proc = subprocess.Popen(['zstd' if archive.suffix == '.zst' else 'gzip', '-dc', '--', str(archive)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, start_new_session=True)
|
|
try:
|
|
self.remember_child(proc)
|
|
with tarfile.open(fileobj=proc.stdout, mode='r|') as tar:
|
|
for item in tar:
|
|
self.disk()
|
|
unpacked += ((item.size + 4095) // 4096) * 4096 + 4096
|
|
count += 1
|
|
while proc.stdout.read(1024*1024):
|
|
self.disk()
|
|
if proc.wait() != 0:
|
|
raise TestError('Archive decompression failed')
|
|
return unpacked, count
|
|
finally:
|
|
proc.stdout.close()
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
proc.wait()
|
|
(self.root / 'child.json').unlink(missing_ok=True)
|
|
|
|
def database(self, sql):
|
|
self.check_image()
|
|
# No imported code can reach a host DB socket, production path or network.
|
|
data = self.scratch / 'database'
|
|
data.mkdir(mode=0o700)
|
|
(self.root / 'db-created').write_text(self.container + '\n')
|
|
startup = '''set -eu
|
|
mkdir -p /var/lib/mysql/tmp
|
|
mariadb-install-db --no-defaults --datadir=/var/lib/mysql/db --auth-root-authentication-method=normal --skip-test-db --user=root --tmpdir=/var/lib/mysql/tmp >/dev/null 2>&1
|
|
exec mariadbd --no-defaults --user=root --datadir=/var/lib/mysql/db --socket=/var/lib/mysql/mysql.sock --pid-file=/var/lib/mysql/mysql.pid --tmpdir=/var/lib/mysql/tmp --skip-networking --skip-log-bin --event-scheduler=OFF --local-infile=0 --innodb-buffer-pool-size=64M --innodb-log-file-size=16M --max-allowed-packet=64M
|
|
'''
|
|
self.command(DOCKER + ['run', '-d', '--pull=never', '--name', self.container,
|
|
'--label', f'{OWNER_LABEL}={self.owner}', '--network=none', '--read-only',
|
|
'--cap-drop=ALL', '--security-opt=no-new-privileges', '--user=0:0',
|
|
'--memory=512m', '--memory-swap=512m', '--cpus=1', '--pids-limit=128',
|
|
'--log-driver=none', '--no-healthcheck', '--volume', str(data) + ':/var/lib/mysql:Z',
|
|
'--entrypoint=/bin/sh', self.image_id, '-c', startup])
|
|
client = DOCKER + ['exec', '-i', self.container, 'mariadb', '--no-defaults',
|
|
'--socket=/var/lib/mysql/mysql.sock', '-uroot', '--batch', '--skip-column-names']
|
|
ready = time.monotonic() + 90
|
|
while True:
|
|
try:
|
|
self.command(client + ['--execute=SELECT 1'], capture=64, timeout=10)
|
|
break
|
|
except TestError:
|
|
self.disk()
|
|
if time.monotonic() > ready:
|
|
raise TestError('Isolated MariaDB did not become ready') from None
|
|
time.sleep(0.5)
|
|
self.command(client + ['--execute=CREATE DATABASE restore_test'])
|
|
with sql.open('rb') as stream:
|
|
# --binary-mode disables client commands in batch SQL (e.g. system/source).
|
|
self.command(client + ['--binary-mode', 'restore_test'], stdin=stream, timeout=self.args.timeout)
|
|
self.command(DOCKER + ['exec', self.container, 'mariadb-check', '--no-defaults',
|
|
'--socket=/var/lib/mysql/mysql.sock', '-uroot', '--all-databases'], timeout=self.args.timeout)
|
|
tables = self.command(client + ["--execute=SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')"], capture=128).decode().strip()
|
|
return int(tables)
|
|
|
|
def execute(self):
|
|
self.preflight()
|
|
manifest = self.select()
|
|
self.log.info('Testing %s, %s components, sequentially', self.state['run'], len(manifest['archives']))
|
|
for item in manifest['archives']:
|
|
component = dict(label=item['label'], status='RUNNING', archive_bytes=item['size'])
|
|
self.state['components'].append(component)
|
|
self.save()
|
|
self.log.info('Component %s: download, verify, extract', item['label'])
|
|
self.disk(additional=item['size'], force=True)
|
|
archive = self.scratch / item['file']
|
|
with archive.open('wb') as output:
|
|
self.remote('cat', self.args.remote.rstrip('/') + '/' + self.state['run'] + '/' + item['file'],
|
|
output=output, limit=item['size'])
|
|
if archive.stat().st_size != item['size'] or backup_set.digest(archive) != item['sha256']:
|
|
raise TestError('Archive checksum/size mismatch: ' + item['label'])
|
|
# Run shared validation as a monitored child, so even corrupt streams
|
|
# and metadata-heavy archives cannot bypass the total timeout.
|
|
self.command([sys.executable, str(Path(__file__).with_name('backup_set.py')),
|
|
'tar', str(archive), item['root']], timeout=self.args.timeout)
|
|
unpacked, entries = self.expand_size(archive)
|
|
self.disk(additional=unpacked, force=True)
|
|
extracted = self.scratch / 'extracted'
|
|
extracted.mkdir(mode=0o700)
|
|
self.command(['tar', '--extract', '--file', str(archive), '--acls', '--xattrs', '--selinux',
|
|
'--numeric-owner', '--same-owner', '-C', str(extracted)], timeout=self.args.timeout)
|
|
# The compressed input is no longer needed; free it BEFORE DB import.
|
|
archive.unlink()
|
|
component.update(extracted_bytes=unpacked, entries=entries)
|
|
if item['label'].startswith('db-'):
|
|
sql = extracted / item['root']
|
|
if not sql.is_file() or sql.is_symlink() or not sql.stat().st_size:
|
|
raise TestError('SQL dump missing or empty')
|
|
self.disk(additional=512*1024*1024, force=True)
|
|
component['tables'] = self.database(sql)
|
|
self.monitor = False
|
|
self.cleanup()
|
|
self.scratch.mkdir(mode=0o700)
|
|
self.monitor = True
|
|
component['status'] = 'SUCCESS'
|
|
self.save()
|
|
self.log.info('Component %s: SUCCESS; files and test DB removed', item['label'])
|
|
self.state['status'] = 'SUCCESS'
|
|
|
|
def report(self):
|
|
fqdn = socket.getfqdn()
|
|
host = self.args.hostname or fqdn
|
|
state = self.state
|
|
message = EmailMessage(policy=SMTP)
|
|
message['From'] = self.args.sender or 'app-backup@' + fqdn
|
|
message['To'] = self.args.to
|
|
message['Subject'] = f"[app-restore-test] {state['status']} {host} {state['run']}"
|
|
message['Date'] = dt.datetime.now().astimezone()
|
|
body = ['Restore-Test', f'Host: {host}', f"Status: {state['status']}", f"Sicherung: {state['run']}",
|
|
f"Alter: {state.get('backup_age_hours', '?')} Stunden", f"Dauer: {state['duration_seconds']}s",
|
|
f"Bereinigung: {state['cleanup']}", f"Fehler: {state['error']}", '', 'Komponenten:']
|
|
for c in state['components']:
|
|
body.append(f"{c['label']}: {c['status']}; Archiv {c['archive_bytes']} Bytes; "
|
|
f"Einträge {c.get('entries', '?')}; Tabellen {c.get('tables', '-')}")
|
|
body.extend(['', 'Dateien entpackt; enthaltene DB-Dumps isoliert importiert und Tabellen geprüft.',
|
|
'Dies ist kein Start-/Funktionstest der produktiven Anwendungen.'])
|
|
message.set_content('\n'.join(body))
|
|
raw = message.as_bytes()
|
|
(self.root / 'report.eml').write_bytes(raw)
|
|
if self.args.mail == 'true':
|
|
if not self.args.to.strip() or not str(message['From']).strip():
|
|
raise TestError('Empty mail recipient or sender')
|
|
with io_bytes(raw) as stream:
|
|
self.command([self.args.sendmail, '-oi', '-t'], stdin=stream, timeout=self.args.mail_timeout)
|
|
|
|
def finish(self):
|
|
self.monitor = False
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(sig, signal.SIG_IGN)
|
|
try:
|
|
self.cleanup()
|
|
except Exception as exc:
|
|
self.state.update(status='FAIL', cleanup='FAILED', error='Cleanup failed: ' + str(exc))
|
|
self.log.error('%s', self.state['error'])
|
|
for c in self.state['components']:
|
|
if c['status'] == 'RUNNING':
|
|
c['status'] = 'FAIL'
|
|
self.state['duration_seconds'] = max(0, int(time.time() - self.state['started_epoch']))
|
|
self.state['finished_epoch'] = int(time.time())
|
|
self.save()
|
|
if self.state['status'] == 'SUCCESS':
|
|
atomic_json(self.root / 'last-success.json', self.state)
|
|
try:
|
|
self.report()
|
|
self.state['mail'] = 'SENT' if self.args.mail == 'true' else 'DISABLED'
|
|
except Exception as exc:
|
|
self.state['mail'] = 'FAILED'
|
|
self.log.error('Restore-test report failed (%s); inspect state.json', type(exc).__name__)
|
|
self.save()
|
|
self.log.info('Restore-test %s; cleanup %s', self.state['status'], self.state['cleanup'])
|
|
return 0 if self.state['status'] == 'SUCCESS' and self.state['mail'] != 'FAILED' else 1
|
|
|
|
|
|
def io_bytes(raw):
|
|
# A real temporary FD for Popen stdin, unlinked immediately on creation.
|
|
import tempfile
|
|
stream = tempfile.TemporaryFile()
|
|
stream.write(raw)
|
|
stream.seek(0)
|
|
return stream
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('mode', choices=('run', 'check', 'cleanup'))
|
|
for key in ('workdir', 'prefix', 'remote', 'run', 'image', 'hostname', 'to', 'sender', 'sendmail', 'mail'):
|
|
parser.add_argument('--' + key, required=True)
|
|
for key in ('remote-check', 'config'):
|
|
parser.add_argument('--' + key)
|
|
parser.add_argument('--expected', action='append', default=[])
|
|
for key in ('max-age-hours', 'timeout', 'mail-timeout'):
|
|
parser.add_argument('--' + key, type=int, required=True)
|
|
for key in ('max-work-gb', 'min-free-gb'):
|
|
parser.add_argument('--' + key, type=float, required=True)
|
|
args = parser.parse_args()
|
|
if not (0 < args.max_work_gb <= 100000 and 0 <= args.min_free_gb <= 100000 and
|
|
1 <= args.max_age_hours <= 8760 and 1 <= args.timeout <= 86400 and 1 <= args.mail_timeout <= 300):
|
|
parser.error('Invalid restore-test limits')
|
|
if os.geteuid() != 0:
|
|
parser.error('Run as root via app-restore-test.sh')
|
|
test = Rehearsal(args)
|
|
if args.mode == 'cleanup' and not test.root.exists():
|
|
return 0
|
|
if args.mode == 'check':
|
|
if any((test.root / name).exists() for name in ('scratch', 'db-created', 'child.json')):
|
|
raise TestError('Interrupted test data found; run --cleanup before --check')
|
|
test.preflight()
|
|
test.log.info('Restore-test prerequisites and disk reserve OK; no image pulled or backup downloaded')
|
|
return 0
|
|
previous = test.root / 'state.json'
|
|
if previous.exists():
|
|
try:
|
|
old = json.loads(previous.read_text())
|
|
if not isinstance(old, dict) or not {'status', 'cleanup', 'started_epoch', 'components', 'run'} <= old.keys():
|
|
raise ValueError('Invalid state')
|
|
except (ValueError, OSError):
|
|
old = dict(test.state)
|
|
if old.get('status') == 'RUNNING' or old.get('cleanup') != 'OK':
|
|
test.state = old
|
|
test.state.update(status='FAIL', error='Previous restore test interrupted; recovery cleanup')
|
|
result = test.finish()
|
|
if args.mode == 'cleanup' or test.state['cleanup'] != 'OK':
|
|
return result
|
|
test = Rehearsal(args)
|
|
if args.mode == 'cleanup':
|
|
test.cleanup()
|
|
return 0
|
|
def interrupted(sig, _):
|
|
raise TestError('Interrupted by signal ' + str(sig))
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(sig, interrupted)
|
|
try:
|
|
test.cleanup()
|
|
test.state['cleanup'] = 'PENDING'
|
|
test.save()
|
|
test.scratch.mkdir(mode=0o700)
|
|
test.execute()
|
|
except Exception as exc:
|
|
test.state.update(status='FAIL', error=str(exc))
|
|
test.log.error('Restore-test failed: %s', exc)
|
|
return test.finish()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
os.umask(0o077)
|
|
try:
|
|
sys.exit(main())
|
|
except Exception as exc:
|
|
print(f'ERROR: restore-test {type(exc).__name__}: {exc}', file=sys.stderr)
|
|
sys.exit(1)
|