Harden backup integrity, implement verified restore and safe upgrades
This commit is contained in:
admin committed
1 parent
099688ea48
commit
4d58fc7b66
22 files changed
+1928
-1106
No files matched your search
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict backup-set metadata and tar validation; never execute backup metadata.
|
||||
|
||||
GNU tar performs extraction to preserve ACLs/xattrs. This helper first validates
|
||||
all member names and link relationships, independent of archive member order.
|
||||
"""
|
||||
import configparser
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
FORMAT = 2
|
||||
LABEL = re.compile(r"(?:meta|wordpress|nextcloud(?:-data)?|gitea(?:-etc|-repos|-custom)?|mail|postfix|dovecot|mail-home-\d+|extra-\d+|db-(?:wordpress|nextcloud|gitea))\Z")
|
||||
RUN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*_(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})_[0-9a-f]{8}\Z")
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def regular(path):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError(f"Not a regular file: {path}")
|
||||
|
||||
|
||||
def expected_root(label):
|
||||
if label == "meta":
|
||||
return "meta"
|
||||
if label.startswith("db-"):
|
||||
return f"db/{label[3:]}.sql"
|
||||
return f"files/{label}"
|
||||
|
||||
|
||||
def load_set(folder, metadata_only=False):
|
||||
regular(folder / "manifest.json")
|
||||
regular(folder / "COMPLETE")
|
||||
raw = (folder / "manifest.json").read_bytes()
|
||||
if len(raw) > 1024 * 1024:
|
||||
raise ValueError("Manifest too large")
|
||||
if (folder / "COMPLETE").read_text() != f"APP_BACKUP_V2 {hashlib.sha256(raw).hexdigest()}\n":
|
||||
raise ValueError("Missing or mismatched completion marker")
|
||||
data = json.loads(raw)
|
||||
if data["format"] != FORMAT or not RUN.fullmatch(data["run_id"]):
|
||||
raise ValueError("Unsupported backup format/run ID")
|
||||
if not isinstance(data["created_epoch"], int) or data["created_epoch"] <= 0:
|
||||
raise ValueError("Invalid creation time")
|
||||
seen = set()
|
||||
if not isinstance(data["archives"], list) or not data["archives"]:
|
||||
raise ValueError("Empty backup set")
|
||||
for item in data["archives"]:
|
||||
label = item["label"]
|
||||
if not LABEL.fullmatch(label) or label in seen:
|
||||
raise ValueError(f"Invalid/duplicate component: {label}")
|
||||
seen.add(label)
|
||||
if item["root"] != expected_root(label):
|
||||
raise ValueError("Unexpected archive root")
|
||||
if item["file"] not in (f"{label}.tar.gz", f"{label}.tar.zst"):
|
||||
raise ValueError("Unsafe archive filename")
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]):
|
||||
raise ValueError("Invalid SHA256")
|
||||
if not isinstance(item["size"], int) or item["size"] <= 0:
|
||||
raise ValueError("Invalid archive size")
|
||||
if not metadata_only:
|
||||
archive = folder / item["file"]
|
||||
regular(archive)
|
||||
if archive.stat().st_size != item["size"] or digest(archive) != item["sha256"]:
|
||||
raise ValueError(f"Archive checksum/size mismatch: {archive.name}")
|
||||
if "meta" not in seen:
|
||||
raise ValueError("Missing metadata component")
|
||||
if not metadata_only:
|
||||
allowed = {a["file"] for a in data["archives"]} | {"manifest.json", "COMPLETE", "UPLOADED"}
|
||||
if {p.name for p in folder.iterdir()} - allowed:
|
||||
raise ValueError("Unexpected files in backup set")
|
||||
return data
|
||||
|
||||
|
||||
def member_name(value):
|
||||
path = PurePosixPath(value)
|
||||
if path.is_absolute() or ".." in path.parts or not path.parts:
|
||||
raise ValueError(f"Unsafe tar path: {value!r}")
|
||||
if any(ord(c) < 32 for c in value):
|
||||
raise ValueError("Control character in tar path")
|
||||
return str(path)
|
||||
|
||||
|
||||
def check_tar(path, roots):
|
||||
command = ["zstd", "-dc", "--", str(path)] if str(path).endswith(".zst") else ["gzip", "-dc", "--", str(path)]
|
||||
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
|
||||
entries = {}
|
||||
hardlinks = []
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
|
||||
for item in archive:
|
||||
name = member_name(item.name)
|
||||
if not any(name == root or name.startswith(root + "/") for root in roots):
|
||||
raise ValueError(f"Member outside component root: {name}")
|
||||
if name in entries:
|
||||
raise ValueError(f"Duplicate tar member: {name}")
|
||||
if not (item.isdir() or item.isfile() or item.issym() or item.islnk()):
|
||||
raise ValueError(f"Special device/FIFO/socket not supported: {name}")
|
||||
entries[name] = "dir" if item.isdir() else "file" if item.isfile() else "link"
|
||||
if item.islnk():
|
||||
hardlinks.append((name, member_name(item.linkname)))
|
||||
# Symlinks may point outside the archive (e.g. /etc/alternatives),
|
||||
# but no member may be extracted THROUGH a symlink or hardlink.
|
||||
for name in entries:
|
||||
for parent in PurePosixPath(name).parents:
|
||||
if str(parent) in entries and entries[str(parent)] != "dir":
|
||||
raise ValueError(f"Tar member descends through a non-directory: {name}")
|
||||
for name, target in hardlinks:
|
||||
if entries.get(target) != "file":
|
||||
raise ValueError(f"Hardlink target is not an archived regular file: {name}")
|
||||
if any(entries.get(root) != ("file" if root.endswith(".sql") else "dir") for root in roots):
|
||||
raise ValueError("Required archive root missing")
|
||||
# Consume the compression trailer, including errors after the tar EOF.
|
||||
while proc.stdout.read(1024 * 1024):
|
||||
pass
|
||||
if proc.wait() != 0:
|
||||
raise ValueError(f"Decompression failed: {path}")
|
||||
finally:
|
||||
proc.stdout.close()
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
|
||||
|
||||
def create(folder, run_id, plan):
|
||||
entries = []
|
||||
for line in Path(plan).read_text().splitlines():
|
||||
label, source = line.split("\t")
|
||||
matches = list(folder.glob(f"{label}.tar.*"))
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"Missing/ambiguous archive: {label}")
|
||||
path = matches[0]
|
||||
check_tar(path, [expected_root(label)])
|
||||
entries.append(dict(label=label, source=source, root=expected_root(label), file=path.name,
|
||||
size=path.stat().st_size, sha256=digest(path)))
|
||||
manifest = dict(format=FORMAT, run_id=run_id, created_epoch=int(dt.datetime.now(dt.timezone.utc).timestamp()),
|
||||
hostname=os.uname().nodename, archives=entries)
|
||||
(folder / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=True) + "\n")
|
||||
(folder / "COMPLETE").write_text(f"APP_BACKUP_V2 {digest(folder / 'manifest.json')}\n")
|
||||
|
||||
|
||||
def crypt(name):
|
||||
cfg = configparser.ConfigParser(interpolation=None)
|
||||
try:
|
||||
cfg.read_string(sys.stdin.read())
|
||||
except configparser.Error:
|
||||
raise ValueError("Invalid rclone configuration (details suppressed)") from None
|
||||
remote = cfg[name]
|
||||
if remote.get("type") != "crypt":
|
||||
raise ValueError("Remote must have type=crypt")
|
||||
if remote.get("filename_encryption", "standard") != "standard":
|
||||
raise ValueError("Require filename_encryption=standard")
|
||||
if not remote.getboolean("directory_name_encryption", True):
|
||||
raise ValueError("Directory encryption is disabled")
|
||||
if remote.getboolean("no_data_encryption", False):
|
||||
raise ValueError("Content encryption is disabled")
|
||||
print("Crypt configuration accepted (contents, filenames, directories)")
|
||||
|
||||
|
||||
def candidates(prefix, days, keep):
|
||||
# Only v2 names supplied AFTER completion-marker validation are eligible.
|
||||
names = set(sys.stdin.read().splitlines())
|
||||
valid = []
|
||||
for name in names:
|
||||
match = RUN.fullmatch(name)
|
||||
if match and name.startswith(prefix + "_"):
|
||||
date = dt.datetime.strptime(match[1], "%Y-%m-%d_%H-%M-%S").replace(tzinfo=dt.timezone.utc)
|
||||
valid.append((date, name))
|
||||
valid.sort(reverse=True)
|
||||
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=int(days))
|
||||
for date, name in valid[int(keep):]:
|
||||
if date < cutoff:
|
||||
print(name)
|
||||
|
||||
|
||||
def write_status(folder, operation, run_id, code):
|
||||
data = dict(operation=operation, run=run_id, exit_code=int(code),
|
||||
time=int(dt.datetime.now(dt.timezone.utc).timestamp()))
|
||||
names = [f"last-{operation}.json"]
|
||||
if operation == "backup" and int(code) == 0:
|
||||
names.append("last-successful-backup.json")
|
||||
for name in names:
|
||||
path = Path(folder) / name
|
||||
with open(str(path) + ".tmp", "w") as stream:
|
||||
json.dump(data, stream)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(str(path) + ".tmp", path)
|
||||
|
||||
|
||||
def main():
|
||||
op, *args = sys.argv[1:]
|
||||
if op == "status":
|
||||
write_status(*args)
|
||||
elif op == "crypt":
|
||||
crypt(*args)
|
||||
elif op == "create":
|
||||
create(Path(args[0]), *args[1:])
|
||||
elif op in ("verify", "metadata", "list"):
|
||||
data = load_set(Path(args[0]), metadata_only=op == "metadata")
|
||||
if op == "verify":
|
||||
for item in data["archives"]:
|
||||
check_tar(Path(args[0]) / item["file"], [item["root"]])
|
||||
print(f"Verified: {data['run_id']} ({len(data['archives'])} archives)")
|
||||
elif op == "list":
|
||||
for item in data["archives"]:
|
||||
print("\t".join([item["label"], item["file"], item["root"]]))
|
||||
elif op == "candidates":
|
||||
candidates(*args)
|
||||
elif op == "tar":
|
||||
check_tar(Path(args[0]), args[1:])
|
||||
else:
|
||||
raise ValueError("Unknown operation")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (ValueError, KeyError, OSError, tarfile.TarError, configparser.Error) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
+345
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared safety rules. Configuration is trusted root-owned Bash, never archive input.
|
||||
set -Eeuo pipefail
|
||||
shopt -s inherit_errexit
|
||||
umask 077
|
||||
export LC_ALL=C
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
need() { command -v "$1" >/dev/null || die "Missing tool: $1"; }
|
||||
helper() { python3 "$LIB_DIR/backup_set.py" "$@"; }
|
||||
|
||||
secure_file() {
|
||||
[[ -f "$1" && ! -L "$1" ]] || die "Expected regular file: $1"
|
||||
[[ $(stat -c %u "$1") == 0 ]] || die "File must belong to root: $1"
|
||||
(( (8#$(stat -c %a "$1") & 8#022) == 0 )) || die "File writable by group/others: $1"
|
||||
}
|
||||
|
||||
load_config() {
|
||||
[[ $EUID == 0 ]] || die "Run as root."
|
||||
secure_file "$CONFIG_FILE"
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
: "${WORKDIR:=/var/backups/app-backup}"
|
||||
: "${STAGING_ROOT:=$WORKDIR/staging}" "${ARCHIVE_DIR:=$WORKDIR/archives}"
|
||||
: "${RESTORE_ROOT:=$WORKDIR/restore}" "${LOG_DIR:=/var/log/app-backup}"
|
||||
: "${LOCKFILE:=/run/app-backup.lock}" "${ARCHIVE_PREFIX:=appbackup}"
|
||||
: "${COMPRESSOR:=zstd}" "${ZSTD_LEVEL:=6}" "${ZSTD_THREADS:=2}"
|
||||
: "${LOG_RETENTION_DAYS:=30}"
|
||||
: "${MIN_FREE_GB:=10}" "${LOCAL_RETENTION_DAYS:=7}" "${REMOTE_RETENTION_DAYS:=30}"
|
||||
: "${RETENTION_KEEP_LAST:=3}" "${ENABLE_REMOTE_RETENTION:=false}"
|
||||
: "${ENABLE_UPLOAD:=true}" "${RCLONE_BIN:=rclone}"
|
||||
: "${RCLONE_REMOTE_BASE:=}" "${RCLONE_CONFIG:=/root/.config/rclone/rclone.conf}"
|
||||
: "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}"
|
||||
: "${RCLONE_RETRIES_SLEEP:=30s}" "${RCLONE_TIMEOUT:=1h}" "${RCLONE_CONTIMEOUT:=30s}"
|
||||
: "${RCLONE_TRANSFERS:=2}" "${RCLONE_CHECKERS:=4}" "${RCLONE_ONEDRIVE_CHUNK_SIZE:=80Mi}"
|
||||
: "${RCLONE_BWLIMIT:=0}" "${ENABLE_MAIL_REPORT:=true}" "${MAIL_TO:=johannes}"
|
||||
: "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
|
||||
: "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
|
||||
: "${ENABLE_NEXTCLOUD_MAINTENANCE:=true}" "${NC_OCC_USER:=apache}"
|
||||
: "${ENABLE_GITEA_SERVICE_STOP:=true}" "${GITEA_SERVICE_NAME:=gitea}"
|
||||
: "${GITEA_DATA_DIR:=/var/lib/gitea/data}" "${GITEA_ETC_DIR:=/etc/gitea}"
|
||||
: "${ENABLE_HTTPD_STOP:=true}" "${HTTPD_SERVICE_NAME:=httpd}"
|
||||
: "${ENABLE_PHPFPM_STOP:=true}" "${PHPFPM_SERVICE_NAME:=php-fpm}"
|
||||
: "${MAIL_BACKUP_MODE:=stop}" "${ENABLE_MAIL_HOME_DIRS:=true}"
|
||||
: "${MAIL_DIR:=/var/vmail}" "${POSTFIX_DIR:=/etc/postfix}" "${DOVECOT_DIR:=/etc/dovecot}"
|
||||
: "${RESTORE_STRICT_DELETE:=false}"
|
||||
# Preserve old space-separated lists; arrays additionally support spaces in paths.
|
||||
if ! declare -p MAIL_HOME_DIRS &>/dev/null; then MAIL_HOME_DIRS=(/home/johannes /home/luciana); fi
|
||||
if [[ $(declare -p MAIL_HOME_DIRS) != 'declare -a '* ]]; then
|
||||
read -r -a MAIL_HOME_DIRS <<< "${MAIL_HOME_DIRS[0]}"
|
||||
fi
|
||||
if ! declare -p MAIL_HOME_EXCLUDES &>/dev/null; then
|
||||
MAIL_HOME_EXCLUDES=(OneDrive/ .cache/ .ccache/ .local/share/Trash/)
|
||||
elif [[ $(declare -p MAIL_HOME_EXCLUDES) != 'declare -a '* ]]; then
|
||||
read -r -a MAIL_HOME_EXCLUDES <<< "${MAIL_HOME_EXCLUDES[0]}"
|
||||
fi
|
||||
for v in ENABLE_UPLOAD ENABLE_REMOTE_RETENTION ENABLE_MAIL_REPORT ENABLE_WORDPRESS ENABLE_NEXTCLOUD \
|
||||
ENABLE_NEXTCLOUD_DATA ENABLE_GITEA ENABLE_MAIL ENABLE_DB_DUMPS ENABLE_NEXTCLOUD_MAINTENANCE \
|
||||
ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE; do
|
||||
[[ ${!v} == true || ${!v} == false ]] || die "$v must be true or false"
|
||||
done
|
||||
for v in LOG_RETENTION_DAYS MIN_FREE_GB LOCAL_RETENTION_DAYS REMOTE_RETENTION_DAYS RETENTION_KEEP_LAST ZSTD_LEVEL ZSTD_THREADS; do
|
||||
[[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer"
|
||||
done
|
||||
(( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2"
|
||||
(( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings"
|
||||
[[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX"
|
||||
[[ $COMPRESSOR == zstd || $COMPRESSOR == gzip ]] || die "COMPRESSOR must be zstd or gzip"
|
||||
[[ $MAIL_BACKUP_MODE == stop || $MAIL_BACKUP_MODE == live ]] || die "MAIL_BACKUP_MODE must be stop or live"
|
||||
if [[ -z ${DB_DUMP_BIN:-} ]]; then DB_DUMP_BIN=$(command -v mariadb-dump || command -v mysqldump || true); fi
|
||||
if [[ -z ${DB_CLIENT_BIN:-} ]]; then DB_CLIENT_BIN=$(command -v mariadb || command -v mysql || true); fi
|
||||
}
|
||||
|
||||
guard_workspaces() {
|
||||
local p src resolved other
|
||||
local sources=("${EXTRA_PATHS[@]}")
|
||||
if [[ $ENABLE_WORDPRESS == true ]]; then sources+=("${WP_DIR:-}"); fi
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then sources+=("${NC_DIR:-}" "${NC_DATA_DIR:-}"); fi
|
||||
if [[ $ENABLE_GITEA == true ]]; then sources+=("$GITEA_DATA_DIR" "$GITEA_ETC_DIR" "${GITEA_REPOS_DIR:-}" "${GITEA_CUSTOM_DIR:-}"); fi
|
||||
if [[ $ENABLE_MAIL == true ]]; then
|
||||
sources+=("$MAIL_DIR" "$POSTFIX_DIR" "$DOVECOT_DIR")
|
||||
if [[ $ENABLE_MAIL_HOME_DIRS == true ]]; then sources+=("${MAIL_HOME_DIRS[@]}"); fi
|
||||
fi
|
||||
# Validate the WHOLE plan before mkdir/chmod touches any configured directory.
|
||||
for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
|
||||
resolved=$(realpath -m -- "$p")
|
||||
case $resolved in /|/etc|/usr|/usr/local|/var|/var/lib|/var/log|/var/backups|/home|/root|/tmp|/run|/opt|/srv)
|
||||
die "Use a dedicated working directory, not $resolved" ;;
|
||||
esac
|
||||
for src in "${sources[@]}"; do
|
||||
[[ -n $src ]] || continue
|
||||
other=$(realpath -m -- "$src")
|
||||
[[ $resolved != "$other" && $resolved != "$other/"* && $other != "$resolved/"* ]] || die "Source and working path overlap: $src / $p"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
init_runtime() {
|
||||
need python3; need flock
|
||||
guard_workspaces
|
||||
# Both backup and restore, and the installer, take the same lock.
|
||||
[[ $LOCKFILE == /* && ! -L $LOCKFILE ]] || die "Unsafe lock path"
|
||||
if [[ -e $LOCKFILE ]]; then
|
||||
[[ -f $LOCKFILE && ! -s $LOCKFILE && $(stat -c %u "$LOCKFILE") == 0 ]] || die "Unsafe existing lock file"
|
||||
fi
|
||||
exec 9>"$LOCKFILE"
|
||||
flock -n 9 || die "Another backup/restore/update holds $LOCKFILE"
|
||||
for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
|
||||
[[ $p == /* && $p != / && $p != *$'\n'* && $p != *$'\t'* ]] || die "Unsafe working path: $p"
|
||||
[[ ! -L $p ]] || die "Symlink working directory refused: $p"
|
||||
mkdir -p -- "$p"
|
||||
[[ $(stat -c %u "$p") == 0 ]] || die "Working directory must belong to root: $p"
|
||||
chmod 700 -- "$p"
|
||||
done
|
||||
RUN_ID="${ARCHIVE_PREFIX}_$(date -u +%Y-%m-%d_%H-%M-%S)_$(python3 -c 'import secrets; print(secrets.token_hex(4))')"
|
||||
LOG_FILE="$LOG_DIR/${OPERATION}_${RUN_ID}.log"
|
||||
# Journald already captures service stdout. No dependency on systemd-cat in terminals.
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
echo "== $OPERATION $RUN_ID =="
|
||||
need rsync; need tar; need "$COMPRESSOR"
|
||||
STOPPED_SERVICES=(); NC_CHANGED=false; NC_WAS_ON=false; MUTATION_STARTED=false
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
trap finish EXIT
|
||||
}
|
||||
|
||||
# Same canonical paths must not contain one another when backup writes into them.
|
||||
check_source_path() {
|
||||
local src p
|
||||
src=$(realpath -e -- "$1") || die "Missing source: $1"
|
||||
[[ -d $src && $src != / ]] || die "Source must be a directory other than /: $1"
|
||||
[[ $src != *$'\n'* && $src != *$'\t'* ]] || die "Unsupported control character in path"
|
||||
for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do
|
||||
p=$(realpath -m -- "$p")
|
||||
[[ $p != "$src" && $p != "$src/"* && $src != "$p/"* ]] || die "Source and working path overlap: $src / $p"
|
||||
done
|
||||
}
|
||||
|
||||
# Build a stable component mapping. Restore destinations come ONLY from local config.
|
||||
build_plan() {
|
||||
LABELS=(); SOURCES=(); ROOTS=(); DB_LABELS=(); DB_NAMES=(); DB_CNFS=()
|
||||
if [[ $ENABLE_WORDPRESS == true ]]; then add_source wordpress "${WP_DIR:?Set WP_DIR}"; add_db wordpress WP; fi
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then
|
||||
add_source nextcloud "${NC_DIR:?Set NC_DIR}"
|
||||
: "${NC_DATA_DIR:=$NC_DIR/data}"
|
||||
if [[ $ENABLE_NEXTCLOUD_DATA == true ]]; then add_source nextcloud-data "$NC_DATA_DIR"; fi
|
||||
add_db nextcloud NC
|
||||
fi
|
||||
if [[ $ENABLE_GITEA == true ]]; then
|
||||
add_source gitea "$GITEA_DATA_DIR"; add_source gitea-etc "$GITEA_ETC_DIR"
|
||||
[[ -z ${GITEA_REPOS_DIR:-} ]] || add_source gitea-repos "$GITEA_REPOS_DIR"
|
||||
[[ -z ${GITEA_CUSTOM_DIR:-} ]] || add_source gitea-custom "$GITEA_CUSTOM_DIR"
|
||||
add_db gitea GITEA
|
||||
fi
|
||||
if [[ $ENABLE_MAIL == true ]]; then
|
||||
add_source mail "$MAIL_DIR"; add_source postfix "$POSTFIX_DIR"; add_source dovecot "$DOVECOT_DIR"
|
||||
if [[ $ENABLE_MAIL_HOME_DIRS == true ]]; then
|
||||
local i=0
|
||||
for p in "${MAIL_HOME_DIRS[@]}"; do add_source "mail-home-$i" "$p"; i=$((i+1)); done
|
||||
fi
|
||||
fi
|
||||
local i=0
|
||||
for p in "${EXTRA_PATHS[@]}"; do add_source "extra-$i" "$p"; i=$((i+1)); done
|
||||
(( ${#LABELS[@]} > 0 )) || die "No enabled sources"
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then
|
||||
local nc_path data_path
|
||||
nc_path=$(realpath -m -- "$NC_DIR"); data_path=$(realpath -m -- "$NC_DATA_DIR")
|
||||
[[ $nc_path != "$data_path" && $nc_path != "$data_path/"* ]] || die 'Nextcloud data must not contain the code directory'
|
||||
if [[ $ENABLE_WORDPRESS == true ]]; then
|
||||
[[ $(realpath -m -- "$WP_DIR") != "$nc_path" ]] || die 'WordPress and Nextcloud must not share the same directory'
|
||||
fi
|
||||
fi
|
||||
}
|
||||
add_source() { LABELS+=("$1"); SOURCES+=("$2"); ROOTS+=("files/$1"); }
|
||||
add_db() {
|
||||
[[ $ENABLE_DB_DUMPS == true ]] || return 0
|
||||
local n="${2}_DB_NAME" c="${2}_DB_CNF"
|
||||
[[ -n ${!n:-} && -n ${!c:-} ]] || die "Set $n and $c for enabled application"
|
||||
[[ ${!n} =~ ^[a-zA-Z0-9_][a-zA-Z0-9_-]*$ ]] || die "Unsupported database name: ${!n}"
|
||||
DB_LABELS+=("db-$1"); DB_NAMES+=("${!n}"); DB_CNFS+=("${!c}")
|
||||
}
|
||||
|
||||
check_db_credentials() {
|
||||
local f
|
||||
for f in "${DB_CNFS[@]}"; do
|
||||
secure_file "$f"
|
||||
(( (8#$(stat -c %a "$f") & 8#077) == 0 )) || die "Use chmod 600 for DB credentials: $f"
|
||||
done
|
||||
}
|
||||
|
||||
preflight() {
|
||||
for p in "${SOURCES[@]}"; do check_source_path "$p"; done
|
||||
check_db_credentials
|
||||
if (( ${#DB_LABELS[@]} )); then
|
||||
need "$DB_DUMP_BIN"; need "$DB_CLIENT_BIN"
|
||||
for i in "${!DB_LABELS[@]}"; do
|
||||
"$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --execute='SELECT 1' >/dev/null
|
||||
done
|
||||
fi
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then need sudo; need php; [[ -f $NC_DIR/occ ]] || die "Nextcloud occ missing"; fi
|
||||
need systemctl
|
||||
if [[ $ENABLE_UPLOAD == true ]]; then check_remote; fi
|
||||
check_space "$STAGING_ROOT"; check_space "$ARCHIVE_DIR"
|
||||
}
|
||||
check_space() {
|
||||
local avail
|
||||
avail=$(df -PB1 -- "$1" | awk 'END{print $4}')
|
||||
(( avail >= MIN_FREE_GB * 1024 * 1024 * 1024 )) || die "Free-space reserve (${MIN_FREE_GB} GiB) unavailable on $1"
|
||||
}
|
||||
|
||||
check_remote() {
|
||||
need "$RCLONE_BIN"
|
||||
[[ $RCLONE_REMOTE_BASE =~ ^[a-zA-Z0-9_-]+:[^:]+$ ]] || die "Use a named crypt remote with a dedicated non-empty path"
|
||||
[[ $RCLONE_REMOTE_BASE != *$'\n'* ]] || die "Invalid remote path"
|
||||
secure_file "$RCLONE_CONFIG"
|
||||
(( (8#$(stat -c %a "$RCLONE_CONFIG") & 8#077) == 0 )) || die "Use chmod 600 for rclone config"
|
||||
# Environment-defined backend types/settings could invalidate a config-only audit.
|
||||
local key name=${RCLONE_REMOTE_BASE%%:*}
|
||||
for key in ${!RCLONE_CONFIG_@}; do
|
||||
[[ $key == RCLONE_CONFIG_PASS ]] || die "Remote overrides through $key are unsupported; use the config file"
|
||||
done
|
||||
# Pipe directly into the parser: compatibility with older rclone without
|
||||
# config redacted. Secret values never go to the log or a temporary file.
|
||||
"$RCLONE_BIN" --config "$RCLONE_CONFIG" config show "$name" | helper crypt "$name"
|
||||
RCLONE_ARGS=(--config "$RCLONE_CONFIG" --no-check-certificate=false --crypt-filename-encryption standard
|
||||
--crypt-directory-name-encryption=true --crypt-no-data-encryption=false
|
||||
--retries "$RCLONE_RETRIES" --low-level-retries "$RCLONE_LOW_LEVEL_RETRIES"
|
||||
--retries-sleep "$RCLONE_RETRIES_SLEEP" --timeout "$RCLONE_TIMEOUT" --contimeout "$RCLONE_CONTIMEOUT"
|
||||
--transfers "$RCLONE_TRANSFERS" --checkers "$RCLONE_CHECKERS" --onedrive-chunk-size "$RCLONE_ONEDRIVE_CHUNK_SIZE"
|
||||
--bwlimit "$RCLONE_BWLIMIT")
|
||||
rc lsd "${name}:" --max-depth 1 >/dev/null
|
||||
}
|
||||
rc() { "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@"; }
|
||||
|
||||
# Exclusions are rooted so a nested directory named data is not accidentally lost.
|
||||
exclusions() {
|
||||
EXCLUDES=()
|
||||
local rel x
|
||||
if [[ $1 == wordpress ]]; then
|
||||
for x in "${NC_DIR:-}" "${NC_DATA_DIR:-}"; do
|
||||
[[ -n $x ]] || continue
|
||||
rel=$(realpath -m --relative-to="${WP_DIR}" -- "$x")
|
||||
if [[ $rel != .. && $rel != ../* && $rel != . ]]; then EXCLUDES+=("--exclude=/$rel/***"); fi
|
||||
done
|
||||
for x in "${WP_EXCLUDES[@]}"; do EXCLUDES+=("--exclude=$x"); done
|
||||
elif [[ $1 == nextcloud ]]; then
|
||||
rel=$(realpath -m --relative-to="$NC_DIR" -- "$NC_DATA_DIR")
|
||||
if [[ $rel != .. && $rel != ../* && $rel != . ]]; then EXCLUDES+=("--exclude=/$rel/***"); fi
|
||||
elif [[ $1 == mail-home-* ]]; then
|
||||
EXCLUDES+=(--one-file-system)
|
||||
for x in "${MAIL_HOME_EXCLUDES[@]}"; do EXCLUDES+=("--exclude=${x#--exclude=}"); done
|
||||
fi
|
||||
}
|
||||
|
||||
stop_service() {
|
||||
local state
|
||||
state=$(systemctl show "$1" --property=ActiveState --value)
|
||||
case $state in
|
||||
active|activating|reloading)
|
||||
# Record intent before stop: a failed/partial stop still needs recovery.
|
||||
STOPPED_SERVICES+=("$1")
|
||||
save_recovery
|
||||
systemctl stop "$1"
|
||||
;;
|
||||
inactive|failed) ;;
|
||||
*) die "Unknown/transitioning service state for $1: $state" ;;
|
||||
esac
|
||||
}
|
||||
occ() { sudo -u "$NC_OCC_USER" php "$NC_DIR/occ" "$@"; }
|
||||
quiesce() {
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then
|
||||
if [[ $ENABLE_NEXTCLOUD_MAINTENANCE == true ]]; then
|
||||
local status
|
||||
status=$(occ status --output=json)
|
||||
NC_WAS_ON=$(printf '%s' "$status" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert isinstance(d["maintenance"], bool); print(str(d["maintenance"]).lower())')
|
||||
if [[ $NC_WAS_ON == false ]]; then NC_CHANGED=true; save_recovery; occ maintenance:mode --on; fi
|
||||
else echo "WARNING: Nextcloud maintenance disabled; consistency depends on external writer shutdown"; fi
|
||||
fi
|
||||
# Include application cron/worker services in this array; crond itself is not stopped.
|
||||
for p in "${QUIESCE_SERVICES[@]}"; do stop_service "$p"; done
|
||||
if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then
|
||||
if [[ $ENABLE_HTTPD_STOP == true ]]; then stop_service "$HTTPD_SERVICE_NAME"; fi
|
||||
if [[ $ENABLE_PHPFPM_STOP == true ]]; then stop_service "$PHPFPM_SERVICE_NAME"; fi
|
||||
fi
|
||||
if [[ $ENABLE_GITEA == true ]]; then
|
||||
if [[ $ENABLE_GITEA_SERVICE_STOP == true ]]; then stop_service "$GITEA_SERVICE_NAME"; else echo "WARNING: Gitea is not stopped"; fi
|
||||
fi
|
||||
if [[ $ENABLE_MAIL == true ]]; then
|
||||
if [[ $MAIL_BACKUP_MODE == stop ]]; then stop_service postfix; stop_service dovecot
|
||||
else echo "WARNING: live mail copy is not a consistent snapshot"; fi
|
||||
fi
|
||||
}
|
||||
# Survives SIGKILL/power loss, which Bash traps cannot handle. This is an
|
||||
# instruction record for the operator, not an automatically executed recovery.
|
||||
save_recovery() {
|
||||
python3 - "$WORKDIR/recovery-$RUN_ID.json" "$OPERATION" "$NC_CHANGED" "$NC_WAS_ON" "${STAGING_DIR:-}" "${STOPPED_SERVICES[@]}" <<'PYREC'
|
||||
import json,os,sys
|
||||
path,operation,changed,was_on,staging,*services=sys.argv[1:]
|
||||
with open(path+'.tmp','w') as stream:
|
||||
json.dump(dict(operation=operation, staging=staging, nextcloud_changed=changed=='true', nextcloud_was_on=was_on=='true', stopped_services=services),stream,indent=2)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(path+'.tmp',path)
|
||||
PYREC
|
||||
}
|
||||
resume_services() {
|
||||
local failed=0 i
|
||||
if [[ $NC_CHANGED == true ]]; then
|
||||
if occ maintenance:mode --off; then NC_CHANGED=false; else failed=1; fi
|
||||
fi
|
||||
for ((i=${#STOPPED_SERVICES[@]}-1; i>=0; i--)); do
|
||||
[[ -n ${STOPPED_SERVICES[i]} ]] || continue
|
||||
if systemctl start "${STOPPED_SERVICES[i]}"; then STOPPED_SERVICES[i]=''; else failed=1; fi
|
||||
done
|
||||
if (( failed == 0 )); then
|
||||
rm -f -- "$WORKDIR/recovery-$RUN_ID.json"
|
||||
else save_recovery; fi
|
||||
return "$failed"
|
||||
}
|
||||
finish() {
|
||||
local code=$?
|
||||
trap - EXIT INT TERM
|
||||
set +e
|
||||
if [[ $OPERATION == restore && $MUTATION_STARTED == true && $code != 0 ]]; then
|
||||
if [[ $ENABLE_NEXTCLOUD == true ]]; then occ maintenance:mode --on || echo 'ERROR: Could not enforce Nextcloud maintenance'; fi
|
||||
save_recovery
|
||||
echo "ERROR: Partial restore. Service recovery requires manual review: ${STOPPED_SERVICES[*]}"
|
||||
echo "ERROR: Keep Nextcloud in maintenance; review $LOG_FILE before restarting."
|
||||
else
|
||||
resume_services || code=1
|
||||
fi
|
||||
if [[ $code == 0 && -n ${STAGING_DIR:-} ]]; then rm -rf -- "$STAGING_DIR" || code=1; fi
|
||||
if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi
|
||||
# Persist status separately from --check; failed status writes also fail the job.
|
||||
if [[ ${CHECK:-false} != true ]]; then
|
||||
helper status "$WORKDIR" "$OPERATION" "$RUN_ID" "$code" || code=1
|
||||
fi
|
||||
if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
|
||||
if [[ -x /usr/sbin/sendmail && $MAIL_TO != *$'\n'* && $MAIL_TO != *$'\r'* ]]; then
|
||||
{ printf 'To: %s\nSubject: [app-backup] exit=%s %s\nContent-Type: text/plain; charset=UTF-8\n\nLog: %s\n' "$MAIL_TO" "$code" "$RUN_ID" "$LOG_FILE"; tail -n 80 "$LOG_FILE"; } |
|
||||
/usr/sbin/sendmail -t || echo "WARNING: report delivery failed"
|
||||
else echo "WARNING: mail report unavailable (sendmail/recipient)"; fi
|
||||
fi
|
||||
exit "$code"
|
||||
}
|
||||
Reference in new issue
Block a user