#!/usr/bin/env bash # Shared safety rules. Configuration is trusted root-owned Bash, never archive input. set -Eeuo pipefail shopt -s inherit_errexit umask 077 export LC_ALL=C die() { echo "ERROR: $*" >&2; exit 1; } need() { command -v "$1" >/dev/null || die "Missing tool: $1"; } helper() { python3 "$LIB_DIR/backup_set.py" "$@"; } secure_file() { [[ -f "$1" && ! -L "$1" ]] || die "Expected regular file: $1" [[ $(stat -c %u "$1") == 0 ]] || die "File must belong to root: $1" (( (8#$(stat -c %a "$1") & 8#022) == 0 )) || die "File writable by group/others: $1" } load_config() { [[ $EUID == 0 ]] || die "Run as root." secure_file "$CONFIG_FILE" # shellcheck disable=SC1090 source "$CONFIG_FILE" : "${WORKDIR:=/var/backups/app-backup}" : "${STAGING_ROOT:=$WORKDIR/staging}" "${ARCHIVE_DIR:=$WORKDIR/archives}" : "${RESTORE_ROOT:=$WORKDIR/restore}" "${LOG_DIR:=/var/log/app-backup}" : "${LOCKFILE:=/run/app-backup.lock}" "${ARCHIVE_PREFIX:=appbackup}" : "${COMPRESSOR:=zstd}" "${ZSTD_LEVEL:=6}" "${ZSTD_THREADS:=2}" : "${LOG_RETENTION_DAYS:=30}" : "${MIN_FREE_GB:=10}" "${LOCAL_RETENTION_DAYS:=7}" "${REMOTE_RETENTION_DAYS:=30}" : "${RETENTION_KEEP_LAST:=3}" "${ENABLE_REMOTE_RETENTION:=false}" : "${ENABLE_UPLOAD:=true}" "${RCLONE_BIN:=rclone}" : "${RCLONE_REMOTE_BASE:=}" "${RCLONE_CONFIG:=/root/.config/rclone/rclone.conf}" : "${RCLONE_RETRIES:=6}" "${RCLONE_LOW_LEVEL_RETRIES:=20}" : "${RCLONE_RETRIES_SLEEP:=30s}" "${RCLONE_TIMEOUT:=1h}" "${RCLONE_CONTIMEOUT:=30s}" : "${RCLONE_TRANSFERS:=2}" "${RCLONE_CHECKERS:=4}" "${RCLONE_ONEDRIVE_CHUNK_SIZE:=80Mi}" : "${RCLONE_BWLIMIT:=0}" "${ENABLE_MAIL_REPORT:=true}" "${MAIL_TO:=johannes}" : "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}" : "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}" : "${ENABLE_NEXTCLOUD_MAINTENANCE:=true}" "${NC_OCC_USER:=apache}" : "${ENABLE_GITEA_SERVICE_STOP:=true}" "${GITEA_SERVICE_NAME:=gitea}" : "${GITEA_DATA_DIR:=/var/lib/gitea/data}" "${GITEA_ETC_DIR:=/etc/gitea}" : "${ENABLE_HTTPD_STOP:=true}" "${HTTPD_SERVICE_NAME:=httpd}" : "${ENABLE_PHPFPM_STOP:=true}" "${PHPFPM_SERVICE_NAME:=php-fpm}" : "${MAIL_BACKUP_MODE:=stop}" "${ENABLE_MAIL_HOME_DIRS:=true}" "${ENABLE_MAIL_STORE:=true}" : "${MAIL_DIR:=/var/vmail}" "${POSTFIX_DIR:=/etc/postfix}" "${DOVECOT_DIR:=/etc/dovecot}" : "${RESTORE_STRICT_DELETE:=false}" # Preserve old space-separated lists; arrays additionally support spaces in paths. if ! declare -p MAIL_HOME_DIRS &>/dev/null; then MAIL_HOME_DIRS=(/home/johannes /home/luciana); fi if [[ $(declare -p MAIL_HOME_DIRS) != 'declare -a '* ]]; then read -r -a MAIL_HOME_DIRS <<< "${MAIL_HOME_DIRS[0]}" fi if ! declare -p MAIL_HOME_EXCLUDES &>/dev/null; then MAIL_HOME_EXCLUDES=(OneDrive/ .cache/ .ccache/ .local/share/Trash/) elif [[ $(declare -p MAIL_HOME_EXCLUDES) != 'declare -a '* ]]; then read -r -a MAIL_HOME_EXCLUDES <<< "${MAIL_HOME_EXCLUDES[0]}" fi for v in ENABLE_UPLOAD ENABLE_REMOTE_RETENTION ENABLE_MAIL_REPORT ENABLE_WORDPRESS ENABLE_NEXTCLOUD \ ENABLE_NEXTCLOUD_DATA ENABLE_GITEA ENABLE_MAIL ENABLE_MAIL_STORE ENABLE_DB_DUMPS ENABLE_NEXTCLOUD_MAINTENANCE \ ENABLE_GITEA_SERVICE_STOP ENABLE_HTTPD_STOP ENABLE_PHPFPM_STOP ENABLE_MAIL_HOME_DIRS RESTORE_STRICT_DELETE; do [[ ${!v} == true || ${!v} == false ]] || die "$v must be true or false" done for v in LOG_RETENTION_DAYS MIN_FREE_GB LOCAL_RETENTION_DAYS REMOTE_RETENTION_DAYS RETENTION_KEEP_LAST ZSTD_LEVEL ZSTD_THREADS; do [[ ${!v} =~ ^(0|[1-9][0-9]*)$ ]] || die "$v must be a non-negative integer" done (( RETENTION_KEEP_LAST >= 2 )) || die "RETENTION_KEEP_LAST must be >= 2" (( ZSTD_LEVEL >= 1 && ZSTD_LEVEL <= 19 && ZSTD_THREADS >= 1 )) || die "Invalid zstd settings" [[ $ARCHIVE_PREFIX =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || die "Unsafe ARCHIVE_PREFIX" [[ $COMPRESSOR == zstd || $COMPRESSOR == gzip ]] || die "COMPRESSOR must be zstd or gzip" [[ $MAIL_BACKUP_MODE == stop || $MAIL_BACKUP_MODE == live ]] || die "MAIL_BACKUP_MODE must be stop or live" # A homes-only setup must still contain mailbox data, not just server config. if [[ $ENABLE_MAIL == true && $ENABLE_MAIL_STORE == false ]]; then [[ $ENABLE_MAIL_HOME_DIRS == true && ${#MAIL_HOME_DIRS[@]} -gt 0 ]] || die 'Mail without a central store requires enabled, nonempty MAIL_HOME_DIRS' fi if [[ -z ${DB_DUMP_BIN:-} ]]; then DB_DUMP_BIN=$(command -v mariadb-dump || command -v mysqldump || true); fi if [[ -z ${DB_CLIENT_BIN:-} ]]; then DB_CLIENT_BIN=$(command -v mariadb || command -v mysql || true); fi } guard_workspaces() { local p src resolved other local sources=("${EXTRA_PATHS[@]}") if [[ $ENABLE_WORDPRESS == true ]]; then sources+=("${WP_DIR:-}"); fi if [[ $ENABLE_NEXTCLOUD == true ]]; then sources+=("${NC_DIR:-}" "${NC_DATA_DIR:-}"); fi if [[ $ENABLE_GITEA == true ]]; then sources+=("$GITEA_DATA_DIR" "$GITEA_ETC_DIR" "${GITEA_REPOS_DIR:-}" "${GITEA_CUSTOM_DIR:-}"); fi if [[ $ENABLE_MAIL == true ]]; then if [[ $ENABLE_MAIL_STORE == true ]]; then sources+=("$MAIL_DIR"); fi sources+=("$POSTFIX_DIR" "$DOVECOT_DIR") if [[ $ENABLE_MAIL_HOME_DIRS == true ]]; then sources+=("${MAIL_HOME_DIRS[@]}"); fi fi # Validate the WHOLE plan before mkdir/chmod touches any configured directory. for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do resolved=$(realpath -m -- "$p") case $resolved in /|/etc|/usr|/usr/local|/var|/var/lib|/var/log|/var/backups|/home|/root|/tmp|/run|/opt|/srv) die "Use a dedicated working directory, not $resolved" ;; esac for src in "${sources[@]}"; do [[ -n $src ]] || continue other=$(realpath -m -- "$src") [[ $resolved != "$other" && $resolved != "$other/"* && $other != "$resolved/"* ]] || die "Source and working path overlap: $src / $p" done done } init_runtime() { need python3; need flock guard_workspaces # shellcheck source=lib/locking.sh source "$LIB_DIR/locking.sh" acquire_backup_lock for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do [[ $p == /* && $p != / && $p != *$'\n'* && $p != *$'\t'* ]] || die "Unsafe working path: $p" [[ ! -L $p ]] || die "Symlink working directory refused: $p" mkdir -p -- "$p" [[ $(stat -c %u "$p") == 0 ]] || die "Working directory must belong to root: $p" chmod 700 -- "$p" done RUN_ID="${ARCHIVE_PREFIX}_$(date -u +%Y-%m-%d_%H-%M-%S)_$(python3 -c 'import secrets; print(secrets.token_hex(4))')" LOG_FILE="$LOG_DIR/${OPERATION}_${RUN_ID}.log" # Journald already captures service stdout. No dependency on systemd-cat in terminals. exec > >(tee -a "$LOG_FILE") 2>&1 echo "== $OPERATION $RUN_ID ==" need rsync; need tar; need "$COMPRESSOR" STOPPED_SERVICES=(); NC_CHANGED=false; NC_WAS_ON=false; MUTATION_STARTED=false trap 'exit 130' INT trap 'exit 143' TERM trap finish EXIT } # Same canonical paths must not contain one another when backup writes into them. check_source_path() { local src p src=$(realpath -e -- "$1") || die "Missing source: $1" [[ -d $src && $src != / ]] || die "Source must be a directory other than /: $1" [[ $src != *$'\n'* && $src != *$'\t'* ]] || die "Unsupported control character in path" for p in "$WORKDIR" "$STAGING_ROOT" "$ARCHIVE_DIR" "$RESTORE_ROOT" "$LOG_DIR"; do p=$(realpath -m -- "$p") [[ $p != "$src" && $p != "$src/"* && $src != "$p/"* ]] || die "Source and working path overlap: $src / $p" done } # Build a stable component mapping. Restore destinations come ONLY from local config. build_plan() { LABELS=(); SOURCES=(); ROOTS=(); DB_LABELS=(); DB_NAMES=(); DB_CNFS=() if [[ $ENABLE_WORDPRESS == true ]]; then add_source wordpress "${WP_DIR:?Set WP_DIR}"; add_db wordpress WP; fi if [[ $ENABLE_NEXTCLOUD == true ]]; then add_source nextcloud "${NC_DIR:?Set NC_DIR}" : "${NC_DATA_DIR:=$NC_DIR/data}" if [[ $ENABLE_NEXTCLOUD_DATA == true ]]; then add_source nextcloud-data "$NC_DATA_DIR"; fi add_db nextcloud NC fi if [[ $ENABLE_GITEA == true ]]; then add_source gitea "$GITEA_DATA_DIR"; add_source gitea-etc "$GITEA_ETC_DIR" [[ -z ${GITEA_REPOS_DIR:-} ]] || add_source gitea-repos "$GITEA_REPOS_DIR" [[ -z ${GITEA_CUSTOM_DIR:-} ]] || add_source gitea-custom "$GITEA_CUSTOM_DIR" add_db gitea GITEA fi if [[ $ENABLE_MAIL == true ]]; then if [[ $ENABLE_MAIL_STORE == true ]]; then add_source mail "$MAIL_DIR"; fi add_source postfix "$POSTFIX_DIR"; add_source dovecot "$DOVECOT_DIR" if [[ $ENABLE_MAIL_HOME_DIRS == true ]]; then local i=0 for p in "${MAIL_HOME_DIRS[@]}"; do add_source "mail-home-$i" "$p"; i=$((i+1)); done fi fi local i=0 for p in "${EXTRA_PATHS[@]}"; do add_source "extra-$i" "$p"; i=$((i+1)); done (( ${#LABELS[@]} > 0 )) || die "No enabled sources" if [[ $ENABLE_NEXTCLOUD == true ]]; then local nc_path data_path nc_path=$(realpath -m -- "$NC_DIR"); data_path=$(realpath -m -- "$NC_DATA_DIR") [[ $nc_path != "$data_path" && $nc_path != "$data_path/"* ]] || die 'Nextcloud data must not contain the code directory' if [[ $ENABLE_WORDPRESS == true ]]; then [[ $(realpath -m -- "$WP_DIR") != "$nc_path" ]] || die 'WordPress and Nextcloud must not share the same directory' fi fi } add_source() { LABELS+=("$1"); SOURCES+=("$2"); ROOTS+=("files/$1"); } add_db() { [[ $ENABLE_DB_DUMPS == true ]] || return 0 local n="${2}_DB_NAME" c="${2}_DB_CNF" [[ -n ${!n:-} && -n ${!c:-} ]] || die "Set $n and $c for enabled application" [[ ${!n} =~ ^[a-zA-Z0-9_][a-zA-Z0-9_-]*$ ]] || die "Unsupported database name: ${!n}" DB_LABELS+=("db-$1"); DB_NAMES+=("${!n}"); DB_CNFS+=("${!c}") } check_db_credentials() { local f for f in "${DB_CNFS[@]}"; do secure_file "$f" (( (8#$(stat -c %a "$f") & 8#077) == 0 )) || die "Use chmod 600 for DB credentials: $f" done } preflight() { for p in "${SOURCES[@]}"; do check_source_path "$p"; done check_db_credentials if (( ${#DB_LABELS[@]} )); then need "$DB_DUMP_BIN"; need "$DB_CLIENT_BIN" for i in "${!DB_LABELS[@]}"; do "$DB_CLIENT_BIN" --defaults-extra-file="${DB_CNFS[i]}" --database="${DB_NAMES[i]}" --execute='SELECT 1' >/dev/null # Restore uses different privileges; this prerequisite belongs to dumps. # Metadata only: never briefly lock the live database during --check. if [[ $OPERATION == backup ]]; then helper db-privileges "$DB_CLIENT_BIN" "${DB_CNFS[i]}" "${DB_NAMES[i]}" fi done fi if [[ $ENABLE_NEXTCLOUD == true ]]; then need sudo; need php; [[ -f $NC_DIR/occ ]] || die "Nextcloud occ missing"; fi need systemctl if [[ $ENABLE_UPLOAD == true ]]; then check_remote; fi check_space "$STAGING_ROOT"; check_space "$ARCHIVE_DIR" } check_space() { local avail avail=$(df -PB1 -- "$1" | awk 'END{print $4}') (( avail >= MIN_FREE_GB * 1024 * 1024 * 1024 )) || die "Free-space reserve (${MIN_FREE_GB} GiB) unavailable on $1" } check_remote() { need "$RCLONE_BIN" [[ $RCLONE_REMOTE_BASE =~ ^[a-zA-Z0-9_-]+:[^:]+$ ]] || die "Use a named crypt remote with a dedicated non-empty path" [[ $RCLONE_REMOTE_BASE != *$'\n'* ]] || die "Invalid remote path" secure_file "$RCLONE_CONFIG" (( (8#$(stat -c %a "$RCLONE_CONFIG") & 8#077) == 0 )) || die "Use chmod 600 for rclone config" # Environment-defined backend types/settings could invalidate a config-only audit. local key name=${RCLONE_REMOTE_BASE%%:*} for key in ${!RCLONE_CONFIG_@}; do [[ $key == RCLONE_CONFIG_PASS ]] || die "Remote overrides through $key are unsupported; use the config file" done # Pipe directly into the parser: compatibility with older rclone without # config redacted. Secret values never go to the log or a temporary file. "$RCLONE_BIN" --ask-password=false --config "$RCLONE_CONFIG" config show "$name" | helper crypt "$name" RCLONE_ARGS=(--ask-password=false --config "$RCLONE_CONFIG" --no-check-certificate=false --crypt-filename-encryption standard --crypt-directory-name-encryption=true --crypt-no-data-encryption=false --retries "$RCLONE_RETRIES" --low-level-retries "$RCLONE_LOW_LEVEL_RETRIES" --retries-sleep "$RCLONE_RETRIES_SLEEP" --timeout "$RCLONE_TIMEOUT" --contimeout "$RCLONE_CONTIMEOUT" --transfers "$RCLONE_TRANSFERS" --checkers "$RCLONE_CHECKERS" --onedrive-chunk-size "$RCLONE_ONEDRIVE_CHUNK_SIZE" --bwlimit "$RCLONE_BWLIMIT") rc lsd "${name}:" --max-depth 1 >/dev/null } rc() { "$RCLONE_BIN" "${RCLONE_ARGS[@]}" "$@"; } # Exclusions are rooted so a nested directory named data is not accidentally lost. exclusions() { EXCLUDES=() local rel x if [[ $1 == wordpress ]]; then for x in "${NC_DIR:-}" "${NC_DATA_DIR:-}"; do [[ -n $x ]] || continue rel=$(realpath -m --relative-to="${WP_DIR}" -- "$x") if [[ $rel != .. && $rel != ../* && $rel != . ]]; then EXCLUDES+=("--exclude=/$rel/***"); fi done for x in "${WP_EXCLUDES[@]}"; do EXCLUDES+=("--exclude=$x"); done elif [[ $1 == nextcloud ]]; then rel=$(realpath -m --relative-to="$NC_DIR" -- "$NC_DATA_DIR") if [[ $rel != .. && $rel != ../* && $rel != . ]]; then EXCLUDES+=("--exclude=/$rel/***"); fi elif [[ $1 == mail-home-* ]]; then EXCLUDES+=(--one-file-system) for x in "${MAIL_HOME_EXCLUDES[@]}"; do EXCLUDES+=("--exclude=${x#--exclude=}"); done fi } stop_service() { local state state=$(systemctl show "$1" --property=ActiveState --value) case $state in active|activating|reloading) # Record intent before stop: a failed/partial stop still needs recovery. STOPPED_SERVICES+=("$1") save_recovery echo "-- Stopping service: $1 (waiting for systemd)" systemctl stop "$1" echo "-- Service stopped: $1" ;; inactive|failed) ;; *) die "Unknown/transitioning service state for $1: $state" ;; esac } occ() { sudo -u "$NC_OCC_USER" php "$NC_DIR/occ" "$@"; } quiesce() { if [[ $ENABLE_NEXTCLOUD == true ]]; then if [[ $ENABLE_NEXTCLOUD_MAINTENANCE == true ]]; then local status status=$(occ status --output=json) NC_WAS_ON=$(printf '%s' "$status" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert isinstance(d["maintenance"], bool); print(str(d["maintenance"]).lower())') if [[ $NC_WAS_ON == false ]]; then NC_CHANGED=true; save_recovery; occ maintenance:mode --on; fi else echo "WARNING: Nextcloud maintenance disabled; consistency depends on external writer shutdown"; fi fi # Cron daemons are included only if explicitly configured by the operator. for p in "${QUIESCE_SERVICES[@]}"; do stop_service "$p"; done if [[ $ENABLE_WORDPRESS == true || $ENABLE_NEXTCLOUD == true ]]; then if [[ $ENABLE_HTTPD_STOP == true ]]; then stop_service "$HTTPD_SERVICE_NAME"; fi if [[ $ENABLE_PHPFPM_STOP == true ]]; then stop_service "$PHPFPM_SERVICE_NAME"; fi fi if [[ $ENABLE_GITEA == true ]]; then if [[ $ENABLE_GITEA_SERVICE_STOP == true ]]; then stop_service "$GITEA_SERVICE_NAME"; else echo "WARNING: Gitea is not stopped"; fi fi if [[ $ENABLE_MAIL == true ]]; then if [[ $MAIL_BACKUP_MODE == stop ]]; then stop_service postfix; stop_service dovecot else echo "WARNING: live mail copy is not a consistent snapshot"; fi fi if [[ $ENABLE_NEXTCLOUD == true ]]; then # Cron daemons may leave an already launched PHP child running after stop. # Wait for direct cron.php / occ processes to finish; never kill them blindly. echo '-- Waiting for remaining Nextcloud writers' helper wait-nextcloud "$NC_DIR" "${WRITER_DRAIN_TIMEOUT:=300}" echo '-- Nextcloud writers finished' fi } # Survives SIGKILL/power loss, which Bash traps cannot handle. This is an # instruction record for the operator, not an automatically executed recovery. save_recovery() { python3 - "$WORKDIR/recovery-$RUN_ID.json" "$OPERATION" "$NC_CHANGED" "$NC_WAS_ON" "${STAGING_DIR:-}" "${STOPPED_SERVICES[@]}" <<'PYREC' import json,os,sys path,operation,changed,was_on,staging,*services=sys.argv[1:] with open(path+'.tmp','w') as stream: json.dump(dict(operation=operation, staging=staging, nextcloud_changed=changed=='true', nextcloud_was_on=was_on=='true', stopped_services=services),stream,indent=2) stream.flush() os.fsync(stream.fileno()) os.replace(path+'.tmp',path) PYREC } resume_services() { local failed=0 i if [[ $NC_CHANGED == true ]]; then if occ maintenance:mode --off; then NC_CHANGED=false; else failed=1; fi fi for ((i=${#STOPPED_SERVICES[@]}-1; i>=0; i--)); do [[ -n ${STOPPED_SERVICES[i]} ]] || continue echo "-- Starting service: ${STOPPED_SERVICES[i]} (waiting for systemd)" if systemctl start "${STOPPED_SERVICES[i]}"; then echo "-- Service started: ${STOPPED_SERVICES[i]}" STOPPED_SERVICES[i]='' else failed=1; fi done if (( failed == 0 )); then rm -f -- "$WORKDIR/recovery-$RUN_ID.json" else save_recovery; fi return "$failed" } finish() { local code=$? trap - EXIT INT TERM set +e if [[ $OPERATION == restore && $MUTATION_STARTED == true && $code != 0 ]]; then if [[ $ENABLE_NEXTCLOUD == true ]]; then occ maintenance:mode --on || echo 'ERROR: Could not enforce Nextcloud maintenance'; fi save_recovery echo "ERROR: Partial restore. Service recovery requires manual review: ${STOPPED_SERVICES[*]}" echo "ERROR: Keep Nextcloud in maintenance; review $LOG_FILE before restarting." else resume_services || code=1 fi if [[ $code == 0 && -n ${STAGING_DIR:-} ]]; then rm -rf -- "$STAGING_DIR" || code=1; fi if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi # Persist status separately from --check; failed status writes also fail the job. if [[ ${CHECK:-false} != true ]]; then helper status "$WORKDIR" "$OPERATION" "$RUN_ID" "$code" || code=1 fi if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then if [[ -x /usr/sbin/sendmail && $MAIL_TO != *$'\n'* && $MAIL_TO != *$'\r'* ]]; then { printf 'To: %s\nSubject: [app-backup] exit=%s %s\nContent-Type: text/plain; charset=UTF-8\n\nLog: %s\n' "$MAIL_TO" "$code" "$RUN_ID" "$LOG_FILE"; tail -n 80 "$LOG_FILE"; } | /usr/sbin/sendmail -t || echo "WARNING: report delivery failed" else echo "WARNING: mail report unavailable (sendmail/recipient)"; fi fi exit "$code" }