Add monthly offsite restore tests and automatic cleanup
This commit is contained in:
admin committed
1 parent
64eab1d3c2
commit
f0c761213d
18 files changed
+1386
-16
No files matched your search
+14
-1
@@ -585,7 +585,17 @@ if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
|
||||
|
||||
def test_automatic_package_upgrade_and_combined_rollback(self):
|
||||
self.enable_report()
|
||||
with self.config.open('a') as f: f.write('MAIL_HOSTNAME="gemini.jr-it-services.de"\n')
|
||||
with self.config.open('a') as f:
|
||||
f.write('MAIL_HOSTNAME="gemini.jr-it-services.de"\nENABLE_RESTORE_TEST=true\nRESTORE_TEST_DB_IMAGE="test-image"\nRESTORE_TEST_MIN_FREE_GB=0\n')
|
||||
# The real isolated DB import is covered separately. Updater preflight
|
||||
# only inspects the prepared local image; it must never pull one.
|
||||
docker=self.bin/'docker'
|
||||
docker.write_text('''#!/usr/bin/env python3
|
||||
import json,sys
|
||||
assert sys.argv[1:] == ['--host=unix:///var/run/docker.sock','image','inspect','test-image']
|
||||
print(json.dumps([{'Id':'sha256:test-image','Config':{'Volumes':{'/var/lib/mysql':{}}}}]))
|
||||
''')
|
||||
docker.chmod(0o700)
|
||||
canonical=self.configure_upgrade()
|
||||
original=canonical.read_bytes()
|
||||
package=self.build_package()
|
||||
@@ -600,6 +610,9 @@ if '--list-only' not in sys.argv and '--exclude=/*' not in sys.argv:
|
||||
self.assertIn('MAIL_HOSTNAME="gemini.jr-it-services.de"',canonical.read_text())
|
||||
self.assertIn('OK (Inhalt und Abschlussmarker',body)
|
||||
self.assertEqual(Path('/usr/local/lib/app-backup/report.py').read_bytes(),(REPO/'lib/report.py').read_bytes())
|
||||
self.assertEqual(Path('/usr/local/lib/app-backup/restore_test.py').read_bytes(),(REPO/'lib/restore_test.py').read_bytes())
|
||||
self.assertTrue(Path('/etc/systemd/system/app-restore-test.timer').is_file())
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['app-restore-test.timer'],'active')
|
||||
self.assertTrue((state/'verified-backup-run').is_file())
|
||||
self.assertIn('ENABLE_REMOTE_RETENTION=false',canonical.read_text())
|
||||
self.assertEqual(self.cnf.stat().st_mode & 0o777,0o600)
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Real local crypt/archive rehearsal tests; Docker DB isolation has a separate opt-in test."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from email.parser import BytesParser
|
||||
from email import policy
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / 'lib'))
|
||||
import backup_set
|
||||
import restore_test
|
||||
|
||||
|
||||
@unittest.skipUnless(os.geteuid() == 0 and shutil.which('rclone'), 'run in test container')
|
||||
class RestoreTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix='restore-rehearsal-')
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.base = Path(self.temp.name)
|
||||
self.work = self.base/'work'
|
||||
self.work.mkdir()
|
||||
self.root = self.work/'restore-test'
|
||||
self.sentinel = self.work/'keep-backup.txt'
|
||||
self.sentinel.write_text('real backup must survive')
|
||||
self.config = self.base/'config'
|
||||
self.crypt = self.base/'crypt.conf'
|
||||
password = subprocess.check_output(['rclone','obscure','restore-test-only'],text=True).strip()
|
||||
self.crypt.write_text(f'[crypt]\ntype=crypt\nremote={self.base}/encrypted\npassword={password}\n')
|
||||
self.crypt.chmod(0o600)
|
||||
self.sendmail = self.base/'sendmail'
|
||||
self.sendmail.write_text(f'#!/bin/sh\ncat > "{self.base}/mail.eml"\n')
|
||||
self.sendmail.chmod(0o700)
|
||||
self.config.write_text(f'''WORKDIR="{self.work}"
|
||||
LOG_DIR="{self.base}/logs"
|
||||
LOCKFILE="{self.base}/lock"
|
||||
ENABLE_RESTORE_TEST=true
|
||||
ENABLE_WORDPRESS=true
|
||||
WP_DIR="{self.base}/source"
|
||||
ENABLE_DB_DUMPS=false
|
||||
ENABLE_MAIL=false
|
||||
ENABLE_NEXTCLOUD=false
|
||||
RCLONE_CONFIG="{self.crypt}"
|
||||
RCLONE_REMOTE_BASE="crypt:sets"
|
||||
RCLONE_RETRIES=1
|
||||
RCLONE_LOW_LEVEL_RETRIES=1
|
||||
RESTORE_TEST_MIN_FREE_GB=0
|
||||
RESTORE_TEST_MAX_WORK_GB=1
|
||||
RESTORE_TEST_TIMEOUT=30
|
||||
MAIL_HOSTNAME="gemini.jr-it-services.de"
|
||||
MAIL_TO="backup@example.invalid"
|
||||
MAIL_FROM="app-backup@example.invalid"
|
||||
MAIL_SENDMAIL_BIN="{self.sendmail}"
|
||||
''')
|
||||
self.config.chmod(0o600)
|
||||
self.run = 'appbackup_'+time.strftime('%Y-%m-%d_%H-%M-%S',time.gmtime())+'_12345678'
|
||||
self.folder = self.base/self.run
|
||||
self.folder.mkdir()
|
||||
self.make_archive('meta', 'meta', b'{}')
|
||||
self.make_archive('wordpress', 'files/wordpress', b'expected data\n')
|
||||
self.manifest()
|
||||
self.upload()
|
||||
|
||||
def make_archive(self, label, root, content):
|
||||
with tarfile.open(self.folder/(label+'.tar.gz'),'w:gz') as tar:
|
||||
if not root.endswith('.sql'):
|
||||
entry = tarfile.TarInfo(root)
|
||||
entry.type = tarfile.DIRTYPE
|
||||
entry.mode = 0o700
|
||||
tar.addfile(entry)
|
||||
root += '/payload.txt'
|
||||
entry = tarfile.TarInfo(root)
|
||||
entry.mode = 0o600
|
||||
entry.size = len(content)
|
||||
tar.addfile(entry,io.BytesIO(content))
|
||||
|
||||
def manifest(self):
|
||||
entries = []
|
||||
for archive in sorted(self.folder.glob('*.tar.gz')):
|
||||
label = archive.name.removesuffix('.tar.gz')
|
||||
entries.append(dict(label=label, source='/source', root=backup_set.expected_root(label),
|
||||
file=archive.name,size=archive.stat().st_size,sha256=backup_set.digest(archive)))
|
||||
data = dict(format=2,run_id=self.run,created_epoch=int(time.time()),archives=entries)
|
||||
self.write_manifest(data)
|
||||
|
||||
def write_manifest(self, data):
|
||||
(self.folder/'manifest.json').write_text(json.dumps(data))
|
||||
(self.folder/'COMPLETE').write_text('APP_BACKUP_V2 '+backup_set.digest(self.folder/'manifest.json')+'\n')
|
||||
|
||||
def rc(self,*args):
|
||||
return subprocess.run(['rclone','--config',str(self.crypt),*args],check=True,capture_output=True)
|
||||
|
||||
def upload(self):
|
||||
self.rc('copy',str(self.folder),'crypt:sets/'+self.run,'--ignore-times')
|
||||
|
||||
def launch(self,*args,**kw):
|
||||
return subprocess.Popen(['bash',str(REPO/'usr-local-sbin/app-restore-test.sh'),'--config',str(self.config),*args],
|
||||
stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True,**kw)
|
||||
|
||||
def execute(self,*args,ok=True):
|
||||
p = self.launch(*args)
|
||||
output = p.communicate(timeout=60)[0]
|
||||
self.assertEqual(p.returncode,0 if ok else 1,output)
|
||||
return output
|
||||
|
||||
def clean(self):
|
||||
self.assertFalse((self.root/'scratch').exists())
|
||||
self.assertFalse((self.root/'db-created').exists())
|
||||
self.assertFalse((self.root/'child.json').exists())
|
||||
self.assertEqual(self.sentinel.read_text(),'real backup must survive')
|
||||
|
||||
def state(self):
|
||||
return json.loads((self.root/'state.json').read_text())
|
||||
|
||||
def test_full_crypt_download_extract_report_and_cleanup(self):
|
||||
self.execute()
|
||||
state = self.state()
|
||||
self.assertEqual(state['status'],'SUCCESS')
|
||||
self.assertEqual(state['cleanup'],'OK')
|
||||
self.assertEqual([c['label'] for c in state['components']],['meta','wordpress'])
|
||||
self.assertTrue(all(c['status']=='SUCCESS' for c in state['components']))
|
||||
self.clean()
|
||||
mail = BytesParser(policy=policy.default).parsebytes((self.base/'mail.eml').read_bytes())
|
||||
self.assertIn('[app-restore-test] SUCCESS gemini.jr-it-services.de',str(mail['Subject']))
|
||||
self.assertIn('Bereinigung: OK',mail.get_content())
|
||||
self.assertNotIn('expected data',mail.get_content())
|
||||
self.assertEqual((self.root/'last-success.json').stat().st_mode & 0o777,0o600)
|
||||
self.execute('--cleanup')
|
||||
self.clean()
|
||||
|
||||
def test_stale_completed_backup_fails_and_cleans(self):
|
||||
data=json.loads((self.folder/'manifest.json').read_text())
|
||||
data['created_epoch']-=4*86400
|
||||
self.write_manifest(data)
|
||||
self.upload()
|
||||
self.execute(ok=False)
|
||||
self.assertIn('stale',self.state()['error'])
|
||||
self.clean()
|
||||
|
||||
def test_corrupt_archive_is_not_success_and_cleans(self):
|
||||
(self.folder/'wordpress.tar.gz').write_bytes(b'corrupt')
|
||||
self.upload()
|
||||
self.execute(ok=False)
|
||||
self.assertEqual(self.state()['status'],'FAIL')
|
||||
self.clean()
|
||||
|
||||
def test_disk_budget_refuses_work_and_cleans(self):
|
||||
with self.config.open('a') as f: f.write('RESTORE_TEST_MAX_WORK_GB=0.000001\n')
|
||||
self.execute(ok=False)
|
||||
self.assertIn('budget',self.state()['error'])
|
||||
self.clean()
|
||||
|
||||
def test_reserve_refuses_work_and_cleans(self):
|
||||
with self.config.open('a') as f: f.write('RESTORE_TEST_MIN_FREE_GB=100000\n')
|
||||
self.execute(ok=False)
|
||||
self.assertIn('reserve',self.state()['error'])
|
||||
self.clean()
|
||||
|
||||
def test_missing_configured_component_fails(self):
|
||||
data=json.loads((self.folder/'manifest.json').read_text())
|
||||
data['archives']=[a for a in data['archives'] if a['label']=='meta']
|
||||
self.write_manifest(data)
|
||||
self.upload()
|
||||
self.execute(ok=False)
|
||||
self.assertIn('missing configured',self.state()['error'])
|
||||
self.clean()
|
||||
|
||||
def test_latest_incomplete_set_is_skipped(self):
|
||||
self.rc('mkdir','crypt:sets/appbackup_2099-01-01_00-00-00_abcdef12')
|
||||
self.execute()
|
||||
self.assertEqual(self.state()['run'],self.run)
|
||||
self.clean()
|
||||
|
||||
def test_scratch_symlink_is_never_followed(self):
|
||||
self.root.mkdir(mode=0o700)
|
||||
(self.root/'.owner').write_text(restore_test.MARKER)
|
||||
(self.root/'scratch').symlink_to(self.work,target_is_directory=True)
|
||||
self.execute('--cleanup',ok=False)
|
||||
self.assertEqual(self.sentinel.read_text(),'real backup must survive')
|
||||
self.assertTrue((self.root/'scratch').is_symlink())
|
||||
|
||||
def test_unknown_directory_is_never_deleted(self):
|
||||
self.root.mkdir(mode=0o700)
|
||||
(self.root/'precious').write_text('keep')
|
||||
self.execute('--cleanup',ok=False)
|
||||
self.assertEqual((self.root/'precious').read_text(),'keep')
|
||||
|
||||
def test_check_preserves_interrupted_cleanup_markers(self):
|
||||
self.root.mkdir(mode=0o700)
|
||||
(self.root/'.owner').write_text(restore_test.MARKER)
|
||||
(self.root/'scratch').mkdir()
|
||||
(self.root/'child.json').write_text('{"pid": 99999999, "start": "old"}')
|
||||
self.execute('--check',ok=False)
|
||||
self.assertTrue((self.root/'child.json').exists())
|
||||
self.assertTrue((self.root/'scratch').exists())
|
||||
|
||||
def test_corrupt_state_does_not_block_cleanup(self):
|
||||
self.root.mkdir(mode=0o700)
|
||||
(self.root/'.owner').write_text(restore_test.MARKER)
|
||||
(self.root/'scratch').mkdir()
|
||||
(self.root/'scratch/old-data').write_text('remove')
|
||||
(self.root/'state.json').write_text('incomplete json')
|
||||
self.execute('--cleanup',ok=False)
|
||||
self.clean()
|
||||
self.assertEqual(self.state()['cleanup'],'OK')
|
||||
|
||||
def test_unreachable_docker_does_not_claim_cleanup_success(self):
|
||||
self.root.mkdir(mode=0o700)
|
||||
(self.root/'.owner').write_text(restore_test.MARKER)
|
||||
(self.root/'scratch').mkdir()
|
||||
(self.root/'scratch/database-file').write_text('potentially still in use')
|
||||
(self.root/'db-created').write_text('pending')
|
||||
(self.root/'state.json').write_text(json.dumps(dict(status='RUNNING',cleanup='PENDING',started_epoch=int(time.time()),components=[],run=self.run)))
|
||||
binpath=self.base/'bin'
|
||||
binpath.mkdir()
|
||||
docker=binpath/'docker'
|
||||
docker.write_text('#!/bin/sh\nexit 42\n')
|
||||
docker.chmod(0o700)
|
||||
env=dict(os.environ,PATH=str(binpath)+':'+os.environ['PATH'])
|
||||
p=self.launch('--cleanup',env=env)
|
||||
output=p.communicate(timeout=20)[0]
|
||||
self.assertEqual(p.returncode,1,output)
|
||||
self.assertEqual(self.state()['cleanup'],'FAILED')
|
||||
self.assertTrue((self.root/'scratch/database-file').exists())
|
||||
self.assertTrue((self.root/'db-created').exists())
|
||||
docker.write_text('#!/bin/sh\nexit 0\n')
|
||||
p=self.launch('--cleanup',env=env)
|
||||
output=p.communicate(timeout=20)[0]
|
||||
self.assertEqual(p.returncode,1,output) # Previous run remains failed.
|
||||
self.clean()
|
||||
self.assertEqual(self.state()['cleanup'],'OK')
|
||||
|
||||
def slow_download(self):
|
||||
wrapper = self.base/'slow-rclone'
|
||||
wrapper.write_text('''#!/bin/bash
|
||||
for arg in "$@"; do
|
||||
if [[ $arg == *wordpress.tar.gz ]]; then sleep 20; fi
|
||||
done
|
||||
exec /usr/bin/rclone "$@"
|
||||
''')
|
||||
wrapper.chmod(0o700)
|
||||
with self.config.open('a') as f: f.write(f'RCLONE_BIN="{wrapper}"\n')
|
||||
|
||||
def wait_download(self,p):
|
||||
end=time.monotonic()+20
|
||||
while time.monotonic()<end:
|
||||
if p.poll() is not None:
|
||||
self.fail(p.communicate()[0])
|
||||
if (self.root/'state.json').exists():
|
||||
state=self.state()
|
||||
if len(state['components'])==2 and (self.root/'child.json').exists():
|
||||
return
|
||||
time.sleep(0.05)
|
||||
self.fail('test did not reach download')
|
||||
|
||||
def test_term_cleans_partial_download(self):
|
||||
self.slow_download()
|
||||
p=self.launch()
|
||||
self.wait_download(p)
|
||||
p.send_signal(signal.SIGTERM)
|
||||
output=p.communicate(timeout=20)[0]
|
||||
self.assertEqual(p.returncode,1,output)
|
||||
self.clean()
|
||||
self.assertEqual(self.state()['status'],'FAIL')
|
||||
|
||||
def test_sigkill_recovery_cleans_orphan_and_reports_failure(self):
|
||||
self.slow_download()
|
||||
p=self.launch()
|
||||
self.wait_download(p)
|
||||
p.kill()
|
||||
p.communicate(timeout=30)
|
||||
self.execute('--cleanup',ok=False)
|
||||
self.clean()
|
||||
self.assertEqual(self.state()['status'],'FAIL')
|
||||
self.assertIn('interrupted',self.state()['error'])
|
||||
|
||||
def test_shared_lock_excludes_restore_test(self):
|
||||
lock=self.base/'lock'
|
||||
p=subprocess.Popen(['flock',str(lock),'sleep','20'])
|
||||
try:
|
||||
time.sleep(.2)
|
||||
test=self.launch()
|
||||
output=test.communicate(timeout=10)[0]
|
||||
self.assertEqual(test.returncode,75,output)
|
||||
self.assertFalse(self.root.exists())
|
||||
finally:
|
||||
p.terminate()
|
||||
p.wait()
|
||||
|
||||
def test_unsafe_lock_is_not_treated_as_busy(self):
|
||||
(self.base/'lock').symlink_to(self.sentinel)
|
||||
self.execute(ok=False)
|
||||
self.assertFalse(self.root.exists())
|
||||
self.assertEqual(self.sentinel.read_text(),'real backup must survive')
|
||||
|
||||
def prepare_database(self, sql):
|
||||
image=os.environ['RESTORE_TEST_REAL_DOCKER_IMAGE']
|
||||
self.make_archive('db-wordpress','db/wordpress.sql',sql)
|
||||
self.manifest()
|
||||
self.upload()
|
||||
with self.config.open('a') as f:
|
||||
f.write(f'ENABLE_DB_DUMPS=true\nWP_DB_NAME=production\nWP_DB_CNF=/unused\nRESTORE_TEST_DB_IMAGE="{image}"\nRESTORE_TEST_TIMEOUT=120\n')
|
||||
|
||||
def no_container(self):
|
||||
self.clean()
|
||||
names=subprocess.check_output(restore_test.DOCKER+['ps','-a','--filter',f'label={restore_test.OWNER_LABEL}', '--format','{{.Names}}'],text=True)
|
||||
self.assertNotIn('app-backup-restore-test-'+hashlib.sha256(str(self.root).encode()).hexdigest()[:16],names)
|
||||
|
||||
@unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
|
||||
def test_real_isolated_db_import_and_container_cleanup(self):
|
||||
self.prepare_database(b'CREATE TABLE example (id INT PRIMARY KEY); INSERT INTO example VALUES (1);')
|
||||
self.execute()
|
||||
state=self.state()
|
||||
db=next(c for c in state['components'] if c['label']=='db-wordpress')
|
||||
self.assertEqual(db['tables'],1)
|
||||
self.no_container()
|
||||
|
||||
@unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
|
||||
def test_real_db_sql_failure_cleans_container_and_data(self):
|
||||
self.prepare_database(b'CREATE TABLE example (id INT); invalid SQL contains-secret-value;')
|
||||
self.execute(ok=False)
|
||||
self.no_container()
|
||||
self.assertEqual(self.state()['cleanup'],'OK')
|
||||
self.assertNotIn('contains-secret-value',(self.root/'report.eml').read_text())
|
||||
self.assertNotIn('contains-secret-value',(self.root/'restore-test.log').read_text())
|
||||
|
||||
@unittest.skipUnless(os.environ.get('RESTORE_TEST_REAL_DOCKER_IMAGE'), 'opt-in local Docker integration')
|
||||
def test_real_db_sigkill_recovery_removes_container_and_data(self):
|
||||
self.prepare_database(b'CREATE TABLE example (id INT); SELECT SLEEP(40);')
|
||||
p=self.launch()
|
||||
container='app-backup-restore-test-'+hashlib.sha256(str(self.root).encode()).hexdigest()[:16]
|
||||
end=time.monotonic()+20
|
||||
while time.monotonic()<end:
|
||||
result=subprocess.run(restore_test.DOCKER+['exec',container,'mariadb','--no-defaults','--socket=/var/lib/mysql/mysql.sock','-uroot','--execute=SELECT COUNT(*) FROM restore_test.example'],capture_output=True)
|
||||
if result.returncode==0:
|
||||
break
|
||||
time.sleep(.2)
|
||||
else:
|
||||
p.terminate()
|
||||
self.fail(p.communicate(timeout=20)[0])
|
||||
info=json.loads(subprocess.check_output(restore_test.DOCKER+['inspect',container]))[0]
|
||||
self.assertEqual(info['HostConfig']['NetworkMode'],'none')
|
||||
self.assertTrue(info['HostConfig']['ReadonlyRootfs'])
|
||||
self.assertEqual(info['HostConfig']['CapDrop'],['ALL'])
|
||||
self.assertEqual(info['HostConfig']['LogConfig']['Type'],'none')
|
||||
self.assertEqual(len(info['Mounts']),1)
|
||||
self.assertEqual(info['Mounts'][0]['Source'],str(self.root/'scratch/database'))
|
||||
p.kill()
|
||||
p.communicate(timeout=20)
|
||||
self.execute('--cleanup',ok=False)
|
||||
self.no_container()
|
||||
self.assertIn('interrupted',self.state()['error'])
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user