Add self-contained automated server upgrade package
This commit is contained in:
admin committed
1 parent
4d58fc7b66
commit
8e6c9a058f
14 files changed
+908
-26
No files matched your search
+2
-2
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
for script in lib/common.sh usr-local-sbin/*.sh install.sh tests/run.sh; do bash -n "$script"; done
|
||||
shellcheck -x lib/common.sh usr-local-sbin/*.sh install.sh tests/run.sh
|
||||
for script in lib/*.sh usr-local-sbin/*.sh install.sh update.sh tests/run.sh; do bash -n "$script"; done
|
||||
shellcheck -x lib/*.sh usr-local-sbin/*.sh install.sh update.sh tests/run.sh
|
||||
python3 -m unittest discover -s tests -v
|
||||
+147
-2
@@ -55,6 +55,23 @@ class ArchiveValidation(unittest.TestCase):
|
||||
with self.assertRaises((ValueError, tarfile.TarError)):
|
||||
HELPER.check_tar(path, ['files/wordpress'])
|
||||
|
||||
def test_writer_drain_waits_and_reports_timeout(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
writer=subprocess.Popen(['python3','-c','import time; time.sleep(1)',directory+'/cron.php'])
|
||||
try:
|
||||
started=time.monotonic()
|
||||
HELPER.wait_nextcloud(directory,3)
|
||||
self.assertGreaterEqual(time.monotonic()-started,.5)
|
||||
finally:
|
||||
writer.wait(timeout=5)
|
||||
writer=subprocess.Popen(['python3','-c','import time; time.sleep(5)',directory+'/occ'])
|
||||
try:
|
||||
with self.assertRaisesRegex(ValueError,'writers still active'):
|
||||
HELPER.wait_nextcloud(directory,1)
|
||||
finally:
|
||||
writer.terminate()
|
||||
writer.wait(timeout=5)
|
||||
|
||||
def test_retention_keeps_minimum_and_ignores_v1(self):
|
||||
result = subprocess.run(['python3', str(REPO/'lib/backup_set.py'), 'candidates', 'appbackup', '7', '2'],
|
||||
input='appbackup_2020-01-01_00-00-00_aaaaaaaa\nappbackup_2020-01-02_00-00-00_bbbbbbbb\nappbackup_2020-01-03_00-00-00_cccccccc\nappbackup_2019-01-01_00-00-00\n../evil\n',
|
||||
@@ -119,12 +136,16 @@ with open(os.environ['TEST_EVENTS'],'a') as f: f.write(name+' '+' '.join(args)+'
|
||||
if name=='systemctl':
|
||||
cmd=args[0]
|
||||
service=args[1] if len(args)>1 else ''
|
||||
if cmd=='show': print(s['services'].get(service,'inactive'))
|
||||
if cmd=='show':
|
||||
if '--property=LoadState' in args: print('loaded' if service in s['services'] else 'not-found')
|
||||
elif '--property=Environment' in args or '--property=TriggeredBy' in args: print('')
|
||||
else: print(s['services'].get(service,'inactive'))
|
||||
elif cmd=='is-active': sys.exit(0 if s['services'].get(args[-1])=='active' else 3)
|
||||
elif cmd in ('stop','start'):
|
||||
if os.environ.get('TEST_FAIL_SERVICE')==cmd+':'+service: sys.exit(1)
|
||||
s['services'][service]='active' if cmd=='start' else 'inactive'
|
||||
elif cmd in ('daemon-reload','enable'): pass
|
||||
elif cmd=='enable': s['services'][args[-1]]='active'
|
||||
elif cmd=='daemon-reload': pass
|
||||
elif name=='sudo':
|
||||
os.execvp(args[2],args[2:])
|
||||
elif name=='php':
|
||||
@@ -373,6 +394,130 @@ QUIESCE_SERVICES=(nc-cron)
|
||||
self.run_script('app-backup.sh','--check',ok=False)
|
||||
self.assertEqual(self.wp.stat().st_mode & 0o777,0o755)
|
||||
|
||||
def build_package(self):
|
||||
output=self.base/'package'
|
||||
subprocess.run(['python3',str(REPO/'tools/build-update-package.py'),'--version','integration','--output',str(output)],check=True,capture_output=True)
|
||||
return output/'app-backup-update-integration.run'
|
||||
|
||||
def configure_upgrade(self):
|
||||
self.crypt_config()
|
||||
canonical=Path('/etc/app-backup/app-backup.conf')
|
||||
canonical.parent.mkdir(exist_ok=True)
|
||||
shutil.copyfile(self.config,canonical)
|
||||
canonical.chmod(0o600)
|
||||
# Old installations commonly left credential files group-readable.
|
||||
self.cnf.chmod(0o640)
|
||||
state=json.loads(self.state.read_text())
|
||||
state['services']['app-backup.timer']='active'
|
||||
self.state.write_text(json.dumps(state))
|
||||
return canonical
|
||||
|
||||
def test_automatic_package_upgrade_and_combined_rollback(self):
|
||||
canonical=self.configure_upgrade()
|
||||
original=canonical.read_bytes()
|
||||
package=self.build_package()
|
||||
result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=120)
|
||||
self.assertEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0])
|
||||
self.assertIn('SUCCESS',(state/'result.txt').read_text())
|
||||
self.assertTrue((state/'verified-backup-run').is_file())
|
||||
self.assertIn('ENABLE_REMOTE_RETENTION=false',canonical.read_text())
|
||||
self.assertEqual(self.cnf.stat().st_mode & 0o777,0o600)
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active')
|
||||
rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30)
|
||||
self.assertEqual(rollback.returncode,0,rollback.stdout+rollback.stderr)
|
||||
self.assertEqual(canonical.read_bytes(),original)
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'inactive')
|
||||
|
||||
def test_package_preflight_failure_preserves_live_configuration_and_timer(self):
|
||||
canonical=self.configure_upgrade()
|
||||
original=canonical.read_bytes()
|
||||
package=self.build_package()
|
||||
result=subprocess.run(['bash',str(package),'--foreground'],env=dict(self.env,TEST_FAIL_TOOL='mariadb'),capture_output=True,text=True,timeout=60)
|
||||
self.assertNotEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
self.assertEqual(canonical.read_bytes(),original)
|
||||
self.assertEqual(self.cnf.stat().st_mode & 0o777,0o640)
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active')
|
||||
self.assertNotIn('systemctl stop',self.events.read_text())
|
||||
|
||||
def test_package_failed_backup_keeps_timer_stopped_and_rollback_available(self):
|
||||
canonical=self.configure_upgrade()
|
||||
original=canonical.read_bytes()
|
||||
package=self.build_package()
|
||||
result=subprocess.run(['bash',str(package),'--foreground'],env=dict(self.env,TEST_FAIL_TOOL='mariadb-dump'),capture_output=True,text=True,timeout=120)
|
||||
self.assertNotEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0])
|
||||
self.assertIn('FAILED',(state/'result.txt').read_text())
|
||||
self.assertTrue((state/'rollback-update.sh').is_file())
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'inactive')
|
||||
self.assertFalse((state/'verified-backup-run').exists())
|
||||
rollback=subprocess.run(['bash',str(state/'rollback-update.sh')],env=self.env,capture_output=True,text=True,timeout=30)
|
||||
self.assertEqual(rollback.returncode,0,rollback.stdout+rollback.stderr)
|
||||
self.assertEqual(canonical.read_bytes(),original)
|
||||
|
||||
def test_package_corruption_is_rejected_before_preparation(self):
|
||||
package=self.build_package()
|
||||
original=package.read_bytes()
|
||||
package.write_bytes(original[:-1]+bytes([original[-1]^1]))
|
||||
result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=20)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertIn('Paket-Pruefsumme falsch',result.stderr)
|
||||
self.assertNotIn('systemctl',self.events.read_text())
|
||||
|
||||
def test_inherited_lock_flag_cannot_bypass_another_process(self):
|
||||
import fcntl
|
||||
with (self.base/'lock').open('w') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
result=self.run_script('app-backup.sh','--check',ok=False,APP_BACKUP_INHERIT_LOCK='1')
|
||||
self.assertIn('Inherited lock descriptor',result.stdout)
|
||||
|
||||
def test_detached_package_handoff_finishes_after_starter_exits(self):
|
||||
self.configure_upgrade()
|
||||
launcher=self.bin/'systemd-run'
|
||||
launcher.write_text('''#!/usr/bin/env python3
|
||||
import os,subprocess,sys
|
||||
from pathlib import Path
|
||||
args=sys.argv[1:]
|
||||
assert '--unit=app-backup-upgrade' in args
|
||||
command=args[args.index('/bin/bash'):]
|
||||
state=Path(command[-1])
|
||||
with open(state/'test-worker.log','w') as log:
|
||||
subprocess.Popen(command,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,env=os.environ,close_fds=True,start_new_session=True)
|
||||
''')
|
||||
launcher.chmod(0o755)
|
||||
package=self.build_package()
|
||||
result=subprocess.run(['bash',str(package)],env=self.env,capture_output=True,text=True,timeout=60)
|
||||
self.assertEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
state=Path(result.stdout.split('Update-Sicherung und Protokoll: ')[1].splitlines()[0])
|
||||
for _ in range(120):
|
||||
status=(state/'result.txt').read_text() if (state/'result.txt').exists() else ''
|
||||
if status.startswith(('SUCCESS','FAILED')): break
|
||||
time.sleep(.5)
|
||||
self.assertTrue(status.startswith('SUCCESS'),status+(state/'test-worker.log').read_text())
|
||||
self.assertIn('unabhaengig von SSH',result.stdout)
|
||||
|
||||
def test_cron_inventory_requires_explicit_policy_and_sorts_timer_first(self):
|
||||
canonical=self.configure_upgrade()
|
||||
cron=Path('/etc/cron.d/app-backup-test-nextcloud')
|
||||
cron.parent.mkdir(exist_ok=True)
|
||||
cron.write_text(f'*/5 * * * * root /usr/bin/php -f {self.nc}/cron.php\n')
|
||||
self.addCleanup(cron.unlink)
|
||||
state=json.loads(self.state.read_text())
|
||||
state['services']['crond.service']='active'
|
||||
state['services']['nc-cron.timer']='active'
|
||||
self.state.write_text(json.dumps(state))
|
||||
package=self.build_package()
|
||||
result=subprocess.run(['bash',str(package),'--foreground'],env=self.env,capture_output=True,text=True,timeout=60)
|
||||
self.assertNotEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
self.assertIn('noninteractive',result.stdout+result.stderr)
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['app-backup.timer'],'active')
|
||||
result=subprocess.run(['bash',str(package),'--foreground','--cron-services','nc-cron,crond.service,nc-cron.timer'],env=self.env,capture_output=True,text=True,timeout=120)
|
||||
self.assertEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
text=canonical.read_text()
|
||||
line=[line for line in text.splitlines() if line.startswith('QUIESCE_SERVICES=')][-1]
|
||||
self.assertLess(line.index('nc-cron.timer'),line.index('crond.service'))
|
||||
self.assertEqual(json.loads(self.state.read_text())['services']['crond.service'],'active')
|
||||
|
||||
def crypt_config(self):
|
||||
(self.base/'encrypted').mkdir()
|
||||
password=subprocess.check_output(['/usr/bin/rclone','obscure','integration-test-only'],text=True).strip()
|
||||
|
||||
Reference in new issue
Block a user