367 lines
15 KiB
Python
367 lines
15 KiB
Python
"""Real local crypt/archive rehearsal tests; Docker DB isolation has a separate opt-in test."""
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from email.parser import BytesParser
|
|
from email import policy
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / 'lib'))
|
|
import backup_set
|
|
import restore_test
|
|
|
|
|
|
@unittest.skipUnless(os.geteuid() == 0 and shutil.which('rclone'), 'run in test container')
|
|
class RestoreTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix='restore-rehearsal-')
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.base = Path(self.temp.name)
|
|
self.work = self.base/'work'
|
|
self.work.mkdir()
|
|
self.root = self.work/'restore-test'
|
|
self.sentinel = self.work/'keep-backup.txt'
|
|
self.sentinel.write_text('real backup must survive')
|
|
self.config = self.base/'config'
|
|
self.crypt = self.base/'crypt.conf'
|
|
password = subprocess.check_output(['rclone','obscure','restore-test-only'],text=True).strip()
|
|
self.crypt.write_text(f'[crypt]\ntype=crypt\nremote={self.base}/encrypted\npassword={password}\n')
|
|
self.crypt.chmod(0o600)
|
|
self.sendmail = self.base/'sendmail'
|
|
self.sendmail.write_text(f'#!/bin/sh\ncat > "{self.base}/mail.eml"\n')
|
|
self.sendmail.chmod(0o700)
|
|
self.config.write_text(f'''WORKDIR="{self.work}"
|
|
LOG_DIR="{self.base}/logs"
|
|
LOCKFILE="{self.base}/lock"
|
|
ENABLE_RESTORE_TEST=true
|
|
ENABLE_WORDPRESS=true
|
|
WP_DIR="{self.base}/source"
|
|
ENABLE_DB_DUMPS=false
|
|
ENABLE_MAIL=false
|
|
ENABLE_NEXTCLOUD=false
|
|
RCLONE_CONFIG="{self.crypt}"
|
|
RCLONE_REMOTE_BASE="crypt:sets"
|
|
RCLONE_RETRIES=1
|
|
RCLONE_LOW_LEVEL_RETRIES=1
|
|
RESTORE_TEST_MIN_FREE_GB=0
|
|
RESTORE_TEST_MAX_WORK_GB=1
|
|
RESTORE_TEST_TIMEOUT=30
|
|
MAIL_HOSTNAME="gemini.jr-it-services.de"
|
|
MAIL_TO="backup@example.invalid"
|
|
MAIL_FROM="app-backup@example.invalid"
|
|
MAIL_SENDMAIL_BIN="{self.sendmail}"
|
|
''')
|
|
self.config.chmod(0o600)
|
|
self.run = 'appbackup_'+time.strftime('%Y-%m-%d_%H-%M-%S',time.gmtime())+'_12345678'
|
|
self.folder = self.base/self.run
|
|
self.folder.mkdir()
|
|
self.make_archive('meta', 'meta', b'{}')
|
|
self.make_archive('wordpress', 'files/wordpress', b'expected data\n')
|
|
self.manifest()
|
|
self.upload()
|
|
|
|
def make_archive(self, label, root, content):
|
|
with tarfile.open(self.folder/(label+'.tar.gz'),'w:gz') as tar:
|
|
if not root.endswith('.sql'):
|
|
entry = tarfile.TarInfo(root)
|
|
entry.type = tarfile.DIRTYPE
|
|
entry.mode = 0o700
|
|
tar.addfile(entry)
|
|
root += '/payload.txt'
|
|
entry = tarfile.TarInfo(root)
|
|
entry.mode = 0o600
|
|
entry.size = len(content)
|
|
tar.addfile(entry,io.BytesIO(content))
|
|
|
|
def manifest(self):
|
|
entries = []
|
|
for archive in sorted(self.folder.glob('*.tar.gz')):
|
|
label = archive.name.removesuffix('.tar.gz')
|
|
entries.append(dict(label=label, source='/source', root=backup_set.expected_root(label),
|
|
file=archive.name,size=archive.stat().st_size,sha256=backup_set.digest(archive)))
|
|
data = dict(format=2,run_id=self.run,created_epoch=int(time.time()),archives=entries)
|
|
self.write_manifest(data)
|
|
|
|
def write_manifest(self, data):
|
|
(self.folder/'manifest.json').write_text(json.dumps(data))
|
|
(self.folder/'COMPLETE').write_text('APP_BACKUP_V2 '+backup_set.digest(self.folder/'manifest.json')+'\n')
|
|
|
|
def rc(self,*args):
|
|
return subprocess.run(['rclone','--config',str(self.crypt),*args],check=True,capture_output=True)
|
|
|
|
def upload(self):
|
|
self.rc('copy',str(self.folder),'crypt:sets/'+self.run,'--ignore-times')
|
|
|
|
def launch(self,*args,**kw):
|
|
return subprocess.Popen(['bash',str(REPO/'usr-local-sbin/app-restore-test.sh'),'--config',str(self.config),*args],
|
|
stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True,**kw)
|
|
|
|
def execute(self,*args,ok=True):
|
|
p = self.launch(*args)
|
|
output = p.communicate(timeout=60)[0]
|
|
self.assertEqual(p.returncode,0 if ok else 1,output)
|
|
return output
|
|
|
|
def clean(self):
|
|
self.assertFalse((self.root/'scratch').exists())
|
|
self.assertFalse((self.root/'db-created').exists())
|
|
self.assertFalse((self.root/'child.json').exists())
|
|
self.assertEqual(self.sentinel.read_text(),'real backup must survive')
|
|
|
|
def state(self):
|
|
return json.loads((self.root/'state.json').read_text())
|
|
|
|
def test_full_crypt_download_extract_report_and_cleanup(self):
|
|
self.execute()
|
|
state = self.state()
|
|
self.assertEqual(state['status'],'SUCCESS')
|
|
self.assertEqual(state['cleanup'],'OK')
|
|
self.assertEqual([c['label'] for c in state['components']],['meta','wordpress'])
|
|
self.assertTrue(all(c['status']=='SUCCESS' for c in state['components']))
|
|
self.clean()
|
|
mail = BytesParser(policy=policy.default).parsebytes((self.base/'mail.eml').read_bytes())
|
|
self.assertIn('[app-restore-test] SUCCESS gemini.jr-it-services.de',str(mail['Subject']))
|
|
self.assertIn('Bereinigung: OK',mail.get_content())
|
|
self.assertNotIn('expected data',mail.get_content())
|
|
self.assertEqual((self.root/'last-success.json').stat().st_mode & 0o777,0o600)
|
|
self.execute('--cleanup')
|
|
self.clean()
|
|
|
|
def test_stale_completed_backup_fails_and_cleans(self):
|
|
data=json.loads((self.folder/'manifest.json').read_text())
|
|
data['created_epoch']-=4*86400
|
|
self.write_manifest(data)
|
|
self.upload()
|
|
self.execute(ok=False)
|
|
self.assertIn('stale',self.state()['error'])
|
|
self.clean()
|
|
|
|
def test_corrupt_archive_is_not_success_and_cleans(self):
|
|
(self.folder/'wordpress.tar.gz').write_bytes(b'corrupt')
|
|
self.upload()
|
|
self.execute(ok=False)
|
|
self.assertEqual(self.state()['status'],'FAIL')
|
|
self.clean()
|
|
|
|
def test_disk_budget_refuses_work_and_cleans(self):
|
|
with self.config.open('a') as f: f.write('RESTORE_TEST_MAX_WORK_GB=0.000001\n')
|
|
self.execute(ok=False)
|
|
self.assertIn('budget',self.state()['error'])
|
|
self.clean()
|
|
|
|
def test_reserve_refuses_work_and_cleans(self):
|
|
with self.config.open('a') as f: f.write('RESTORE_TEST_MIN_FREE_GB=100000\n')
|
|
self.execute(ok=False)
|
|
self.assertIn('reserve',self.state()['error'])
|
|
self.clean()
|
|
|
|
def test_missing_configured_component_fails(self):
|
|
data=json.loads((self.folder/'manifest.json').read_text())
|
|
data['archives']=[a for a in data['archives'] if a['label']=='meta']
|
|
self.write_manifest(data)
|
|
self.upload()
|
|
self.execute(ok=False)
|
|
self.assertIn('missing configured',self.state()['error'])
|
|
self.clean()
|
|
|
|
def test_latest_incomplete_set_is_skipped(self):
|
|
self.rc('mkdir','crypt:sets/appbackup_2099-01-01_00-00-00_abcdef12')
|
|
self.execute()
|
|
self.assertEqual(self.state()['run'],self.run)
|
|
self.clean()
|
|
|
|
def test_scratch_symlink_is_never_followed(self):
|
|
self.root.mkdir(mode=0o700)
|
|
(self.root/'.owner').write_text(restore_test.MARKER)
|
|
(self.root/'scratch').symlink_to(self.work,target_is_directory=True)
|
|
self.execute('--cleanup',ok=False)
|
|
self.assertEqual(self.sentinel.read_text(),'real backup must survive')
|
|
self.assertTrue((self.root/'scratch').is_symlink())
|
|
|
|
def test_unknown_directory_is_never_deleted(self):
|
|
self.root.mkdir(mode=0o700)
|
|
(self.root/'precious').write_text('keep')
|
|
self.execute('--cleanup',ok=False)
|
|
self.assertEqual((self.root/'precious').read_text(),'keep')
|
|
|
|
def test_check_preserves_interrupted_cleanup_markers(self):
|
|
self.root.mkdir(mode=0o700)
|
|
(self.root/'.owner').write_text(restore_test.MARKER)
|
|
(self.root/'scratch').mkdir()
|
|
(self.root/'child.json').write_text('{"pid": 99999999, "start": "old"}')
|
|
self.execute('--check',ok=False)
|
|
self.assertTrue((self.root/'child.json').exists())
|
|
self.assertTrue((self.root/'scratch').exists())
|
|
|
|
def test_corrupt_state_does_not_block_cleanup(self):
|
|
self.root.mkdir(mode=0o700)
|
|
(self.root/'.owner').write_text(restore_test.MARKER)
|
|
(self.root/'scratch').mkdir()
|
|
(self.root/'scratch/old-data').write_text('remove')
|
|
(self.root/'state.json').write_text('incomplete json')
|
|
self.execute('--cleanup',ok=False)
|
|
self.clean()
|
|
self.assertEqual(self.state()['cleanup'],'OK')
|
|
|
|
def test_unreachable_docker_does_not_claim_cleanup_success(self):
|
|
self.root.mkdir(mode=0o700)
|
|
(self.root/'.owner').write_text(restore_test.MARKER)
|
|
(self.root/'scratch').mkdir()
|
|
(self.root/'scratch/database-file').write_text('potentially still in use')
|
|
(self.root/'db-created').write_text('pending')
|
|
(self.root/'state.json').write_text(json.dumps(dict(status='RUNNING',cleanup='PENDING',started_epoch=int(time.time()),components=[],run=self.run)))
|
|
binpath=self.base/'bin'
|
|
binpath.mkdir()
|
|
docker=binpath/'docker'
|
|
docker.write_text('#!/bin/sh\nexit 42\n')
|
|
docker.chmod(0o700)
|
|
env=dict(os.environ,PATH=str(binpath)+':'+os.environ['PATH'])
|
|
p=self.launch('--cleanup',env=env)
|
|
output=p.communicate(timeout=20)[0]
|
|
self.assertEqual(p.returncode,1,output)
|
|
self.assertEqual(self.state()['cleanup'],'FAILED')
|
|
self.assertTrue((self.root/'scratch/database-file').exists())
|
|
self.assertTrue((self.root/'db-created').exists())
|
|
docker.write_text('#!/bin/sh\nexit 0\n')
|
|
p=self.launch('--cleanup',env=env)
|
|
output=p.communicate(timeout=20)[0]
|
|
self.assertEqual(p.returncode,1,output) # Previous run remains failed.
|
|
self.clean()
|
|
self.assertEqual(self.state()['cleanup'],'OK')
|
|
|
|
def slow_download(self):
|
|
wrapper = self.base/'slow-rclone'
|
|
wrapper.write_text('''#!/bin/bash
|
|
for arg in "$@"; do
|
|
if [[ $arg == *wordpress.tar.gz ]]; then sleep 20; fi
|
|
done
|
|
exec /usr/bin/rclone "$@"
|
|
''')
|
|
wrapper.chmod(0o700)
|
|
with self.config.open('a') as f: f.write(f'RCLONE_BIN="{wrapper}"\n')
|
|
|
|
def wait_download(self,p):
|
|
end=time.monotonic()+20
|
|
while time.monotonic()<end:
|
|
if p.poll() is not None:
|
|
self.fail(p.communicate()[0])
|
|
if (self.root/'state.json').exists():
|
|
state=self.state()
|
|
if len(state['components'])==2 and (self.root/'child.json').exists():
|
|
return
|
|
time.sleep(0.05)
|
|
self.fail('test did not reach download')
|
|
|
|
def test_term_cleans_partial_download(self):
|
|
self.slow_download()
|
|
p=self.launch()
|
|
self.wait_download(p)
|
|
p.send_signal(signal.SIGTERM)
|
|
output=p.communicate(timeout=20)[0]
|
|
self.assertEqual(p.returncode,1,output)
|
|
self.clean()
|
|
self.assertEqual(self.state()['status'],'FAIL')
|
|
|
|
def test_sigkill_recovery_cleans_orphan_and_reports_failure(self):
|
|
self.slow_download()
|
|
p=self.launch()
|
|
self.wait_download(p)
|
|
p.kill()
|
|
p.communicate(timeout=30)
|
|
self.execute('--cleanup',ok=False)
|
|
self.clean()
|
|
self.assertEqual(self.state()['status'],'FAIL')
|
|
self.assertIn('interrupted',self.state()['error'])
|
|
|
|
def test_shared_lock_excludes_restore_test(self):
|
|
lock=self.base/'lock'
|
|
p=subprocess.Popen(['flock',str(lock),'sleep','20'])
|
|
try:
|
|
time.sleep(.2)
|
|
test=self.launch()
|
|
output=test.communicate(timeout=10)[0]
|
|
self.assertEqual(test.returncode,75,output)
|
|
self.assertFalse(self.root.exists())
|
|
finally:
|
|
p.terminate()
|
|
p.wait()
|
|
|
|
def test_unsafe_lock_is_not_treated_as_busy(self):
|
|
(self.base/'lock').symlink_to(self.sentinel)
|
|
self.execute(ok=False)
|
|
self.assertFalse(self.root.exists())
|
|
self.assertEqual(self.sentinel.read_text(),'real backup must survive')
|
|
|
|
def prepare_database(self, sql):
|
|
image=os.environ['RESTORE_TEST_REAL_DOCKER_IMAGE']
|
|
self.make_archive('db-wordpress','db/wordpress.sql',sql)
|
|
self.manifest()
|
|
self.upload()
|
|
with self.config.open('a') as f:
|
|
f.write(f'ENABLE_DB_DUMPS=true\nWP_DB_NAME=production\nWP_DB_CNF=/unused\nRESTORE_TEST_DB_IMAGE="{image}"\nRESTORE_TEST_TIMEOUT=120\n')
|
|
|
|
def no_container(self):
|
|
self.clean()
|
|
names=subprocess.check_output(restore_test.DOCKER+['ps','-a','--filter',f'label={restore_test.OWNER_LABEL}', '--format','{{.Names}}'],text=True)
|
|
self.assertNotIn('app-backup-restore-test-'+hashlib.sha256(str(self.root).encode()).hexdigest()[:16],names)
|
|
|
|
@unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
|
|
def test_real_isolated_db_import_and_container_cleanup(self):
|
|
self.prepare_database(b'CREATE TABLE example (id INT PRIMARY KEY); INSERT INTO example VALUES (1);')
|
|
self.execute()
|
|
state=self.state()
|
|
db=next(c for c in state['components'] if c['label']=='db-wordpress')
|
|
self.assertEqual(db['tables'],1)
|
|
self.no_container()
|
|
|
|
@unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
|
|
def test_real_db_sql_failure_cleans_container_and_data(self):
|
|
self.prepare_database(b'CREATE TABLE example (id INT); invalid SQL contains-secret-value;')
|
|
self.execute(ok=False)
|
|
self.no_container()
|
|
self.assertEqual(self.state()['cleanup'],'OK')
|
|
self.assertNotIn('contains-secret-value',(self.root/'report.eml').read_text())
|
|
self.assertNotIn('contains-secret-value',(self.root/'restore-test.log').read_text())
|
|
|
|
@unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
|
|
def test_real_db_sigkill_recovery_removes_container_and_data(self):
|
|
self.prepare_database(b'CREATE TABLE example (id INT); SELECT SLEEP(40);')
|
|
p=self.launch()
|
|
container='app-backup-restore-test-'+hashlib.sha256(str(self.root).encode()).hexdigest()[:16]
|
|
end=time.monotonic()+20
|
|
while time.monotonic()<end:
|
|
result=subprocess.run(restore_test.DOCKER+['exec',container,'mariadb','--no-defaults','--socket=/var/lib/mysql/mysql.sock','-uroot','--execute=SELECT COUNT(*) FROM restore_test.example'],capture_output=True)
|
|
if result.returncode==0:
|
|
break
|
|
time.sleep(.2)
|
|
else:
|
|
p.terminate()
|
|
self.fail(p.communicate(timeout=20)[0])
|
|
info=json.loads(subprocess.check_output(restore_test.DOCKER+['inspect',container]))[0]
|
|
self.assertEqual(info['HostConfig']['NetworkMode'],'none')
|
|
self.assertTrue(info['HostConfig']['ReadonlyRootfs'])
|
|
self.assertEqual(info['HostConfig']['CapDrop'],['ALL'])
|
|
self.assertEqual(info['HostConfig']['LogConfig']['Type'],'none')
|
|
self.assertEqual(len(info['Mounts']),1)
|
|
self.assertEqual(info['Mounts'][0]['Source'],str(self.root/'scratch/database'))
|
|
p.kill()
|
|
p.communicate(timeout=20)
|
|
self.execute('--cleanup',ok=False)
|
|
self.no_container()
|
|
self.assertIn('interrupted',self.state()['error'])
|
|
|
|
|
|
if __name__=='__main__':
|
|
unittest.main()
|