322 lines
14 KiB
Python
322 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Strict backup-set metadata and tar validation; never execute backup metadata.
|
|
|
|
GNU tar performs extraction to preserve ACLs/xattrs. This helper first validates
|
|
all member names and link relationships, independent of archive member order.
|
|
"""
|
|
import configparser
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import tarfile
|
|
|
|
FORMAT = 2
|
|
LABEL = re.compile(r"(?:meta|wordpress|nextcloud(?:-data)?|gitea(?:-etc|-repos|-custom)?|mail|postfix|dovecot|mail-home-\d+|extra-\d+|db-(?:wordpress|nextcloud|gitea))\Z")
|
|
RUN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*_(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})_[0-9a-f]{8}\Z")
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def regular(path):
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError(f"Not a regular file: {path}")
|
|
|
|
|
|
def expected_root(label):
|
|
if label == "meta":
|
|
return "meta"
|
|
if label.startswith("db-"):
|
|
return f"db/{label[3:]}.sql"
|
|
return f"files/{label}"
|
|
|
|
|
|
def load_set(folder, metadata_only=False):
|
|
regular(folder / "manifest.json")
|
|
regular(folder / "COMPLETE")
|
|
raw = (folder / "manifest.json").read_bytes()
|
|
if len(raw) > 1024 * 1024:
|
|
raise ValueError("Manifest too large")
|
|
if (folder / "COMPLETE").read_text() != f"APP_BACKUP_V2 {hashlib.sha256(raw).hexdigest()}\n":
|
|
raise ValueError("Missing or mismatched completion marker")
|
|
data = json.loads(raw)
|
|
if data["format"] != FORMAT or not RUN.fullmatch(data["run_id"]):
|
|
raise ValueError("Unsupported backup format/run ID")
|
|
if not isinstance(data["created_epoch"], int) or data["created_epoch"] <= 0:
|
|
raise ValueError("Invalid creation time")
|
|
seen = set()
|
|
if not isinstance(data["archives"], list) or not data["archives"]:
|
|
raise ValueError("Empty backup set")
|
|
for item in data["archives"]:
|
|
label = item["label"]
|
|
if not LABEL.fullmatch(label) or label in seen:
|
|
raise ValueError(f"Invalid/duplicate component: {label}")
|
|
seen.add(label)
|
|
if item["root"] != expected_root(label):
|
|
raise ValueError("Unexpected archive root")
|
|
if item["file"] not in (f"{label}.tar.gz", f"{label}.tar.zst"):
|
|
raise ValueError("Unsafe archive filename")
|
|
if not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]):
|
|
raise ValueError("Invalid SHA256")
|
|
if not isinstance(item["size"], int) or item["size"] <= 0:
|
|
raise ValueError("Invalid archive size")
|
|
if not metadata_only:
|
|
archive = folder / item["file"]
|
|
regular(archive)
|
|
if archive.stat().st_size != item["size"] or digest(archive) != item["sha256"]:
|
|
raise ValueError(f"Archive checksum/size mismatch: {archive.name}")
|
|
if "meta" not in seen:
|
|
raise ValueError("Missing metadata component")
|
|
if not metadata_only:
|
|
allowed = {a["file"] for a in data["archives"]} | {"manifest.json", "COMPLETE", "UPLOADED"}
|
|
if {p.name for p in folder.iterdir()} - allowed:
|
|
raise ValueError("Unexpected files in backup set")
|
|
return data
|
|
|
|
|
|
def member_name(value):
|
|
path = PurePosixPath(value)
|
|
if path.is_absolute() or ".." in path.parts or not path.parts:
|
|
raise ValueError(f"Unsafe tar path: {value!r}")
|
|
if any(ord(c) < 32 for c in value):
|
|
raise ValueError("Control character in tar path")
|
|
return str(path)
|
|
|
|
|
|
def check_tar(path, roots):
|
|
command = ["zstd", "-dc", "--", str(path)] if str(path).endswith(".zst") else ["gzip", "-dc", "--", str(path)]
|
|
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
|
|
entries = {}
|
|
hardlinks = []
|
|
expanded = 0
|
|
try:
|
|
with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
|
|
for item in archive:
|
|
expanded += ((item.size + 4095) // 4096) * 4096 + 4096
|
|
name = member_name(item.name)
|
|
if not any(name == root or name.startswith(root + "/") for root in roots):
|
|
raise ValueError(f"Member outside component root: {name}")
|
|
if name in entries:
|
|
raise ValueError(f"Duplicate tar member: {name}")
|
|
if not (item.isdir() or item.isfile() or item.issym() or item.islnk()):
|
|
raise ValueError(f"Special device/FIFO/socket not supported: {name}")
|
|
entries[name] = "dir" if item.isdir() else "file" if item.isfile() else "link"
|
|
if item.islnk():
|
|
hardlinks.append((name, member_name(item.linkname)))
|
|
# Symlinks may point outside the archive (e.g. /etc/alternatives),
|
|
# but no member may be extracted THROUGH a symlink or hardlink.
|
|
for name in entries:
|
|
for parent in PurePosixPath(name).parents:
|
|
if str(parent) in entries and entries[str(parent)] != "dir":
|
|
raise ValueError(f"Tar member descends through a non-directory: {name}")
|
|
for name, target in hardlinks:
|
|
if entries.get(target) != "file":
|
|
raise ValueError(f"Hardlink target is not an archived regular file: {name}")
|
|
if any(entries.get(root) != ("file" if root.endswith(".sql") else "dir") for root in roots):
|
|
raise ValueError("Required archive root missing")
|
|
# Consume the compression trailer, including errors after the tar EOF.
|
|
while proc.stdout.read(1024 * 1024):
|
|
pass
|
|
if proc.wait() != 0:
|
|
raise ValueError(f"Decompression failed: {path}")
|
|
return expanded
|
|
finally:
|
|
proc.stdout.close()
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
proc.wait()
|
|
|
|
|
|
def create(folder, run_id, plan):
|
|
entries = []
|
|
for line in Path(plan).read_text().splitlines():
|
|
label, source = line.split("\t")
|
|
matches = list(folder.glob(f"{label}.tar.*"))
|
|
if len(matches) != 1:
|
|
raise ValueError(f"Missing/ambiguous archive: {label}")
|
|
path = matches[0]
|
|
check_tar(path, [expected_root(label)])
|
|
entries.append(dict(label=label, source=source, root=expected_root(label), file=path.name,
|
|
size=path.stat().st_size, sha256=digest(path)))
|
|
manifest = dict(format=FORMAT, run_id=run_id, created_epoch=int(dt.datetime.now(dt.timezone.utc).timestamp()),
|
|
hostname=os.uname().nodename, archives=entries)
|
|
(folder / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=True) + "\n")
|
|
(folder / "COMPLETE").write_text(f"APP_BACKUP_V2 {digest(folder / 'manifest.json')}\n")
|
|
|
|
|
|
def crypt(name):
|
|
cfg = configparser.ConfigParser(interpolation=None)
|
|
try:
|
|
cfg.read_string(sys.stdin.read())
|
|
except configparser.Error:
|
|
raise ValueError("Invalid rclone configuration (details suppressed)") from None
|
|
remote = cfg[name]
|
|
if remote.get("type") != "crypt":
|
|
raise ValueError("Remote must have type=crypt")
|
|
if remote.get("filename_encryption", "standard") != "standard":
|
|
raise ValueError("Require filename_encryption=standard")
|
|
if not remote.getboolean("directory_name_encryption", True):
|
|
raise ValueError("Directory encryption is disabled")
|
|
if remote.getboolean("no_data_encryption", False):
|
|
raise ValueError("Content encryption is disabled")
|
|
print("Crypt configuration accepted (contents, filenames, directories)")
|
|
|
|
|
|
def has_reload(grants):
|
|
"""Inspect privilege names only; SHOW GRANTS can contain authentication hashes."""
|
|
for line in grants.splitlines():
|
|
match = re.match(r'^GRANT ([A-Z_ ,]+) ON \*\.\* TO ', line)
|
|
if match and {'RELOAD', 'ALL PRIVILEGES'} & set(match[1].split(', ')):
|
|
return True
|
|
return False
|
|
|
|
|
|
def db_privileges(client, credentials, database):
|
|
"""Read privileges without executing FLUSH or acquiring a production DB lock.
|
|
|
|
Check lock-all-tables and MariaDB's routine metadata prerequisite, not every
|
|
object-specific permission. Grants (including auth hashes) stay in memory.
|
|
"""
|
|
def query(sql, purpose='privilege metadata'):
|
|
result = subprocess.run([client, '--defaults-extra-file=' + credentials,
|
|
'--database=' + database, '--batch', '--skip-column-names',
|
|
'--execute=' + sql], capture_output=True, text=True)
|
|
if result.returncode:
|
|
# Neither query results nor server errors may disclose auth material.
|
|
raise ValueError(f'DB {database}: {purpose} query failed; inspect account permissions')
|
|
return result.stdout
|
|
|
|
grants = query('SHOW GRANTS FOR CURRENT_USER')
|
|
allowed = has_reload(grants)
|
|
via_role = False
|
|
mariadb = 'MariaDB' in query('SELECT VERSION()')
|
|
if not allowed and mariadb:
|
|
# Only session-enabled roles count; a merely granted but inactive role
|
|
# does not help mariadb-dump. ENABLED_ROLES also includes nested roles.
|
|
roles = query('SELECT ROLE_NAME FROM information_schema.ENABLED_ROLES WHERE ROLE_NAME IS NOT NULL').splitlines()
|
|
for role in roles:
|
|
if not re.fullmatch(r'[A-Za-z0-9_.:-]+', role):
|
|
raise ValueError(f'DB {database}: unsupported role name; configure direct RELOAD grant for backup account')
|
|
if has_reload(query('SHOW GRANTS FOR `' + role + '`')):
|
|
allowed = via_role = True
|
|
break
|
|
if not allowed:
|
|
raise ValueError(f'DB {database}: global RELOAD privilege missing or not verifiable; '
|
|
'required for --lock-all-tables. Grant RELOAD to the actual backup account; '
|
|
'see UPDATE_ANLEITUNG.md. No FLUSH or database lock performed by this check')
|
|
print(f'DB {database}: global RELOAD privilege confirmed' + (' through an enabled role' if via_role else ''))
|
|
if mariadb:
|
|
# --routines needs mysql.proc access. LIMIT 0 checks permissions without
|
|
# returning routine bodies or reading the stored routine contents.
|
|
query('SELECT * FROM mysql.proc LIMIT 0', 'routine metadata (SELECT on mysql.proc required for --routines)')
|
|
print(f'DB {database}: MariaDB routine metadata access confirmed')
|
|
|
|
|
|
def candidates(prefix, days, keep):
|
|
# Only v2 names supplied AFTER completion-marker validation are eligible.
|
|
names = set(sys.stdin.read().splitlines())
|
|
valid = []
|
|
for name in names:
|
|
match = RUN.fullmatch(name)
|
|
if match and name.startswith(prefix + "_"):
|
|
date = dt.datetime.strptime(match[1], "%Y-%m-%d_%H-%M-%S").replace(tzinfo=dt.timezone.utc)
|
|
valid.append((date, name))
|
|
valid.sort(reverse=True)
|
|
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=int(days))
|
|
for date, name in valid[int(keep):]:
|
|
if date < cutoff:
|
|
print(name)
|
|
|
|
|
|
def write_status(folder, operation, run_id, code):
|
|
data = dict(operation=operation, run=run_id, exit_code=int(code),
|
|
time=int(dt.datetime.now(dt.timezone.utc).timestamp()))
|
|
names = [f"last-{operation}.json"]
|
|
if operation == "backup" and int(code) == 0:
|
|
names.append("last-successful-backup.json")
|
|
for name in names:
|
|
path = Path(folder) / name
|
|
with open(str(path) + ".tmp", "w") as stream:
|
|
json.dump(data, stream)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(str(path) + ".tmp", path)
|
|
|
|
|
|
def wait_nextcloud(directory, timeout):
|
|
timeout = int(timeout)
|
|
if timeout < 1 or timeout > 3600:
|
|
raise ValueError("WRITER_DRAIN_TIMEOUT must be 1..3600 seconds")
|
|
expected = {os.path.realpath(directory + "/cron.php"), os.path.realpath(directory + "/occ")}
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
found = []
|
|
for process in Path("/proc").iterdir():
|
|
if not process.name.isdigit() or int(process.name) == os.getpid():
|
|
continue
|
|
try:
|
|
args = (process / "cmdline").read_bytes().split(b"\0")
|
|
cwd = os.readlink(process / "cwd")
|
|
# Exact script arguments only: do not match this helper's NC_DIR
|
|
# argument, unrelated files, or arbitrary shell command strings.
|
|
if any(arg and os.path.realpath(os.path.join(cwd, os.fsdecode(arg))) in expected for arg in args):
|
|
found.append(process.name)
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
continue
|
|
if not found:
|
|
return
|
|
if time.monotonic() >= deadline:
|
|
raise ValueError("Nextcloud writers still active (PIDs: " + ", ".join(found) + ")")
|
|
time.sleep(1)
|
|
|
|
|
|
def main():
|
|
op, *args = sys.argv[1:]
|
|
if op == "wait-nextcloud":
|
|
wait_nextcloud(*args)
|
|
elif op == "db-privileges":
|
|
db_privileges(*args)
|
|
elif op == "status":
|
|
write_status(*args)
|
|
elif op == "crypt":
|
|
crypt(*args)
|
|
elif op == "create":
|
|
create(Path(args[0]), *args[1:])
|
|
elif op in ("verify", "metadata", "list", "expanded"):
|
|
data = load_set(Path(args[0]), metadata_only=op == "metadata")
|
|
if op == "expanded":
|
|
print(sum(check_tar(Path(args[0]) / item['file'], [item['root']]) for item in data['archives']))
|
|
elif op == "verify":
|
|
for item in data["archives"]:
|
|
check_tar(Path(args[0]) / item["file"], [item["root"]])
|
|
print(f"Verified: {data['run_id']} ({len(data['archives'])} archives)")
|
|
elif op == "list":
|
|
for item in data["archives"]:
|
|
print("\t".join([item["label"], item["file"], item["root"]]))
|
|
elif op == "candidates":
|
|
candidates(*args)
|
|
elif op == "tar":
|
|
check_tar(Path(args[0]), args[1:])
|
|
else:
|
|
raise ValueError("Unknown operation")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (ValueError, KeyError, OSError, tarfile.TarError, configparser.Error) as exc:
|
|
print(f"ERROR: {exc}", file=sys.stderr)
|
|
sys.exit(1)
|