Harden backup integrity, implement verified restore and safe upgrades
This commit is contained in:
admin committed
1 parent
099688ea48
commit
4d58fc7b66
22 files changed
+1928
-1106
No files matched your search
+180
-101
@@ -1,107 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
# Restore is intentionally two-phase: verify/extract everything, then touch targets.
|
||||
set -Eeuo pipefail
|
||||
umask 027
|
||||
|
||||
# ==============================================================================
|
||||
# app-restore.sh
|
||||
# - Restore from a "run folder" (local dir or rclone remote folder)
|
||||
# - Applies archives per component (meta/db/wordpress/nextcloud/nextcloud-data/gitea...)
|
||||
# ==============================================================================
|
||||
|
||||
LOG_DIR="/var/log/app-backup"
|
||||
mkdir -p "$LOG_DIR"
|
||||
ts="$(date '+%Y-%m-%d_%H-%M-%S')"
|
||||
LOG_FILE="${LOG_DIR}/app-restore_${ts}.log"
|
||||
exec > >(tee -a "$LOG_FILE" | systemd-cat -t app-restore -p info) 2>&1
|
||||
|
||||
CONFIG_FILE="/etc/app-backup/app-backup.conf"
|
||||
if [[ -r "$CONFIG_FILE" ]]; then
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
LIB_DIR="$SCRIPT_DIR/../lib/app-backup"
|
||||
[[ -f $LIB_DIR/common.sh ]] || LIB_DIR="$SCRIPT_DIR/../lib"
|
||||
# shellcheck source=lib/common.sh
|
||||
source "$LIB_DIR/common.sh"
|
||||
CONFIG_FILE=/etc/app-backup/app-backup.conf
|
||||
LOCAL_RUN=''; REMOTE_RUN=''; MODE=plan; YES=false; LEGACY=false
|
||||
FILES=true; DATABASES=true; COMPONENTS=''
|
||||
while (( $# )); do
|
||||
case $1 in
|
||||
--config) CONFIG_FILE=${2:?Missing config}; shift 2 ;;
|
||||
--local-run) LOCAL_RUN=${2:?Missing local run}; shift 2 ;;
|
||||
--remote-run) REMOTE_RUN=${2:?Missing remote run}; shift 2 ;;
|
||||
--verify-only) MODE=verify; shift ;;
|
||||
--extract-only) MODE=extract; shift ;;
|
||||
--dry-run) MODE=plan; shift ;;
|
||||
--apply) MODE=apply; shift ;;
|
||||
--yes) YES=true; shift ;;
|
||||
--legacy-unverified) LEGACY=true; shift ;;
|
||||
--no-db) DATABASES=false; shift ;;
|
||||
--no-files) FILES=false; shift ;;
|
||||
--components) COMPONENTS=${2:?Missing comma-separated components}; shift 2 ;;
|
||||
--help)
|
||||
echo 'Usage: app-restore.sh [--config FILE] (--local-run DIR | --remote-run NAME)'
|
||||
echo ' [--dry-run | --verify-only | --extract-only | --apply --yes]'
|
||||
echo ' [--no-db] [--no-files] [--components wordpress,db-wordpress,...]'
|
||||
echo ' Legacy archives: --legacy-unverified --extract-only'
|
||||
exit 0 ;;
|
||||
*) die "Unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
[[ -n $LOCAL_RUN && -z $REMOTE_RUN || -z $LOCAL_RUN && -n $REMOTE_RUN ]] || die 'Choose exactly one backup source'
|
||||
[[ $MODE != apply || $YES == true ]] || die 'Applying a restore requires --apply --yes (overwrites files and database objects)'
|
||||
[[ $LEGACY == false || $MODE == extract ]] || die 'Unverified legacy backups support --extract-only exclusively'
|
||||
OPERATION=restore
|
||||
load_config
|
||||
init_runtime
|
||||
check_space "$RESTORE_ROOT"
|
||||
STAGING_DIR=$(mktemp -d "$RESTORE_ROOT/${RUN_ID}.XXXXXX")
|
||||
INPUT="$STAGING_DIR/input"
|
||||
mkdir -m 700 "$INPUT"
|
||||
if [[ -n $REMOTE_RUN ]]; then
|
||||
[[ $REMOTE_RUN =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]+$ && $REMOTE_RUN == "$ARCHIVE_PREFIX"_* ]] || die 'Unsafe remote run name'
|
||||
check_remote
|
||||
rc copy "${RCLONE_REMOTE_BASE%/}/$REMOTE_RUN" "$INPUT"
|
||||
else
|
||||
echo "ERROR: Config not found/readable: $CONFIG_FILE"
|
||||
exit 2
|
||||
[[ -d $LOCAL_RUN && ! -L $LOCAL_RUN ]] || die 'Local run must be an existing directory'
|
||||
# Resolve overlap separately: archive sources are allowed under WORKDIR, but
|
||||
# recursive copies into the freshly created restore workspace are forbidden.
|
||||
input_real=$(realpath -e "$LOCAL_RUN")
|
||||
stage_real=$(realpath -e "$STAGING_DIR")
|
||||
[[ $stage_real != "$input_real" && $stage_real != "$input_real/"* ]] || die 'Backup source contains restore workspace'
|
||||
rsync -a -- "$LOCAL_RUN/" "$INPUT/"
|
||||
fi
|
||||
|
||||
: "${WORKDIR:=/var/backups/app-backup}"
|
||||
: "${RESTORE_ROOT:=${WORKDIR}/restore}"
|
||||
: "${ARCHIVE_PREFIX:=appbackup}"
|
||||
|
||||
: "${RCLONE_BIN:=rclone}"
|
||||
: "${RCLONE_REMOTE_BASE:=}"
|
||||
|
||||
|
||||
# --- Remote base resolution & crypt enforcement ---
|
||||
# Prefer explicit RCLONE_REMOTE_BASE from config.
|
||||
# If only RCLONE_REMOTE is provided (e.g. "OneDriveCrypt:Sicherung"), derive the host folder.
|
||||
: "${RCLONE_REMOTE:=}"
|
||||
if [[ -z "${RCLONE_REMOTE_BASE}" ]]; then
|
||||
if [[ -n "${RCLONE_REMOTE}" ]]; then
|
||||
RCLONE_REMOTE_BASE="${RCLONE_REMOTE%/}/JRITServerBackups/$(hostname -s)"
|
||||
fi
|
||||
if [[ $LEGACY == true ]]; then
|
||||
echo 'WARNING: Legacy backup has no trusted manifest/completeness proof.'
|
||||
shopt -s nullglob
|
||||
legacy=("$INPUT"/*.tar.gz "$INPUT"/*.tar.zst)
|
||||
(( ${#legacy[@]} )) || die 'No legacy archives found'
|
||||
# Validate the entire batch BEFORE extracting the first file. Legacy roots differ.
|
||||
for archive in "${legacy[@]}"; do
|
||||
[[ ! -L $archive ]] || die 'Symlink archive refused'
|
||||
base=$(basename "$archive")
|
||||
[[ $base =~ ^${ARCHIVE_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}_(meta|db|wordpress|nextcloud|nextcloud-data|gitea|gitea-etc|mail|mail-home|postfix|dovecot)\.tar\.(gz|zst)$ ]] || die "Unknown legacy filename: $base"
|
||||
label=${BASH_REMATCH[1]}
|
||||
case $label in meta|db) root=$label ;; gitea) root=files/gitea-data ;; *) root=files/$label ;; esac
|
||||
helper tar "$archive" "$root"
|
||||
done
|
||||
mkdir "$STAGING_DIR/extracted"
|
||||
for archive in "${legacy[@]}"; do
|
||||
legacy_dest="$STAGING_DIR/extracted/$(basename "$archive")"
|
||||
mkdir "$legacy_dest"
|
||||
tar --extract --file "$archive" --acls --xattrs --selinux --numeric-owner --same-owner -C "$legacy_dest"
|
||||
done
|
||||
echo "Legacy extraction retained for MANUAL review: $STAGING_DIR/extracted"
|
||||
STAGING_DIR=''
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_crypt_remote() {
|
||||
local base="$1"
|
||||
local remote_name="${base%%:*}"
|
||||
[[ -n "${remote_name}" && "${remote_name}" != "${base}" ]] || die "RCLONE_REMOTE_BASE must include an rclone remote prefix like 'OneDriveCrypt:...'. Got: ${base}"
|
||||
have "${RCLONE_BIN}" || die "rclone missing"
|
||||
|
||||
# rclone prints full config; we only need the remote block.
|
||||
local cfg
|
||||
cfg="$("${RCLONE_BIN}" config show "${remote_name}" 2>/dev/null || true)"
|
||||
[[ -n "${cfg}" ]] || die "rclone remote '${remote_name}' not found. Check 'rclone listremotes' and your config."
|
||||
|
||||
local typ
|
||||
typ="$(printf "%s\n" "${cfg}" | awk -F' = ' '$1=="type"{print $2; exit}')"
|
||||
[[ "${typ}" == "crypt" ]] || die "Remote '${remote_name}' is type='${typ:-?}', not 'crypt'. Refusing to restore from non-crypt remote."
|
||||
|
||||
# Enforce filename/directory encryption so OneDrive web does not show cleartext names.
|
||||
local fe de
|
||||
fe="$(printf "%s\n" "${cfg}" | awk -F' = ' '$1=="filename_encryption"{print $2; exit}')"
|
||||
de="$(printf "%s\n" "${cfg}" | awk -F' = ' '$1=="directory_name_encryption"{print $2; exit}')"
|
||||
[[ "${fe}" == "standard" ]] || die "Remote '${remote_name}': filename_encryption='${fe:-?}' (expected 'standard'). Otherwise filenames can appear in cleartext."
|
||||
[[ "${de}" == "true" ]] || die "Remote '${remote_name}': directory_name_encryption='${de:-?}' (expected 'true'). Otherwise directory names can appear in cleartext."
|
||||
}
|
||||
|
||||
: "${DRY_RUN:=false}"
|
||||
: "${RESTORE_DB:=true}"
|
||||
: "${RESTORE_FILES:=true}"
|
||||
: "${RESTORE_STRICT_DELETE:=false}"
|
||||
|
||||
: "${ENABLE_NEXTCLOUD_MAINTENANCE:=true}"
|
||||
: "${NC_OCC_USER:=apache}"
|
||||
: "${NC_FILES_SCAN_AFTER_RESTORE:=false}"
|
||||
|
||||
: "${ENABLE_GITEA_SERVICE_STOP:=true}"
|
||||
: "${GITEA_SERVICE_NAME:=gitea}"
|
||||
|
||||
: "${ENABLE_HTTPD_STOP:=false}"
|
||||
: "${HTTPD_SERVICE_NAME:=httpd}"
|
||||
: "${ENABLE_PHPFPM_STOP:=false}"
|
||||
: "${PHPFPM_SERVICE_NAME:=php-fpm}"
|
||||
|
||||
die() { echo "ERROR: $*"; exit 1; }
|
||||
have() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
run_cmd() { [[ "${DRY_RUN}" == "true" ]] && echo "[DRY_RUN] $*" || "$@"; }
|
||||
|
||||
NC_MAINTENANCE_ON=false
|
||||
nc_maintenance_off() {
|
||||
if [[ "${NC_MAINTENANCE_ON}" == "true" ]]; then
|
||||
echo "-- Nextcloud maintenance mode OFF (trap)..."
|
||||
run_cmd sudo -u "${NC_OCC_USER}" php "${NC_DIR}/occ" maintenance:mode --off || true
|
||||
NC_MAINTENANCE_ON=false
|
||||
helper verify "$INPUT"
|
||||
if [[ $MODE == verify ]]; then echo 'Verification passed; no application changes.'; exit 0; fi
|
||||
helper list "$INPUT" > "$STAGING_DIR/archive-list.tsv"
|
||||
if [[ $MODE == extract ]]; then
|
||||
mkdir "$STAGING_DIR/extracted"
|
||||
while IFS=$'\t' read -r label file _; do
|
||||
mkdir "$STAGING_DIR/extracted/$label"
|
||||
tar --extract --file "$INPUT/$file" --acls --xattrs --selinux --numeric-owner --same-owner -C "$STAGING_DIR/extracted/$label"
|
||||
done < "$STAGING_DIR/archive-list.tsv"
|
||||
echo "Verified extraction retained: $STAGING_DIR/extracted"
|
||||
STAGING_DIR=''
|
||||
exit 0
|
||||
fi
|
||||
build_plan
|
||||
# Selection is validated against both the local configuration and the backup set.
|
||||
SELECTED=(); SELECTED_ROOTS=(); TARGETS=(); IS_DB=(); CNFS=()
|
||||
selected() { [[ -z $COMPONENTS || ",$COMPONENTS," == *",$1,"* ]]; }
|
||||
for i in "${!LABELS[@]}"; do
|
||||
if [[ $FILES == true ]] && selected "${LABELS[i]}"; then
|
||||
SELECTED+=("${LABELS[i]}"); SELECTED_ROOTS+=("${ROOTS[i]}")
|
||||
TARGETS+=("${SOURCES[i]}"); IS_DB+=(false); CNFS+=('')
|
||||
fi
|
||||
}
|
||||
|
||||
GITEA_WAS_STOPPED=false
|
||||
HTTPD_WAS_STOPPED=false
|
||||
PHPFPM_WAS_STOPPED=false
|
||||
gitea_start() { [[ "${GITEA_WAS_STOPPED}" == "true" ]] && { echo "-- Starting gitea (trap)"; run_cmd systemctl start "${GITEA_SERVICE_NAME}" || true; GITEA_WAS_STOPPED=false; }; }
|
||||
httpd_start() { [[ "${HTTPD_WAS_STOPPED}" == "true" ]] && { echo "-- Starting httpd (trap)"; run_cmd systemctl start "${HTTPD_SERVICE_NAME}" || true; HTTPD_WAS_STOPPED=false; }; }
|
||||
phpfpm_start(){ [[ "${PHPFPM_WAS_STOPPED}" == "true" ]] && { echo "-- Starting php-fpm (trap)"; run_cmd systemctl start "${PHPFPM_SERVICE_NAME}" || true; PHPFPM_WAS_STOPPED=false; }; }
|
||||
|
||||
echo "== app-restore done: ${ts} =="
|
||||
echo "-- Working dir: ${RUN_DIR}"
|
||||
echo "-- Log: ${LOG_FILE}"
|
||||
done
|
||||
for i in "${!DB_LABELS[@]}"; do
|
||||
if [[ $DATABASES == true ]] && selected "${DB_LABELS[i]}"; then
|
||||
SELECTED+=("${DB_LABELS[i]}"); SELECTED_ROOTS+=("db/${DB_LABELS[i]#db-}.sql")
|
||||
TARGETS+=("${DB_NAMES[i]}"); IS_DB+=(true); CNFS+=("${DB_CNFS[i]}")
|
||||
fi
|
||||
done
|
||||
(( ${#SELECTED[@]} )) || die 'No components selected'
|
||||
if [[ -n $COMPONENTS ]]; then
|
||||
IFS=, read -r -a requested <<< "$COMPONENTS"
|
||||
for label in "${requested[@]}"; do
|
||||
[[ " ${SELECTED[*]} " == *" $label "* ]] || die "Unknown, disabled or excluded component: $label"
|
||||
done
|
||||
fi
|
||||
for i in "${!SELECTED[@]}"; do
|
||||
label=${SELECTED[i]}
|
||||
awk -F '\t' -v label="$label" '$1==label {found=1} END {exit !found}' "$STAGING_DIR/archive-list.tsv" || die "Required component absent: $label"
|
||||
# Indexed paths must retain their meaning when lists in config are edited.
|
||||
if [[ $label == extra-* || $label == mail-home-* ]]; then
|
||||
python3 - "$INPUT/manifest.json" "$label" "${TARGETS[i]}" <<'PY'
|
||||
import json,sys
|
||||
manifest,label,target=sys.argv[1:]
|
||||
source=next(a['source'] for a in json.load(open(manifest))['archives'] if a['label']==label)
|
||||
if source != target:
|
||||
sys.exit('Indexed source path differs from configuration; restore via --extract-only and review mapping')
|
||||
PY
|
||||
fi
|
||||
echo "RESTORE $label -> ${TARGETS[i]} (database=${IS_DB[i]})"
|
||||
done
|
||||
if [[ $MODE == plan ]]; then
|
||||
echo "Verified restore plan. Strict deletion: $RESTORE_STRICT_DELETE. Use --apply --yes to execute."
|
||||
exit 0
|
||||
fi
|
||||
mkdir "$STAGING_DIR/extracted"
|
||||
# Each component gets an independent extraction root, so links in one component
|
||||
# cannot redirect extraction of a later component into another tree.
|
||||
while IFS=$'\t' read -r label file _; do
|
||||
mkdir "$STAGING_DIR/extracted/$label"
|
||||
tar --extract --file "$INPUT/$file" --acls --xattrs --selinux --numeric-owner --same-owner -C "$STAGING_DIR/extracted/$label"
|
||||
done < "$STAGING_DIR/archive-list.tsv"
|
||||
if [[ $MODE == extract ]]; then
|
||||
echo "Verified extraction retained: $STAGING_DIR/extracted"
|
||||
STAGING_DIR=''
|
||||
exit 0
|
||||
fi
|
||||
# Validate every destination and DB connection before stopping a single service.
|
||||
for i in "${!SELECTED[@]}"; do
|
||||
if [[ ${IS_DB[i]} == true ]]; then
|
||||
secure_file "${CNFS[i]}"
|
||||
(( (8#$(stat -c %a "${CNFS[i]}") & 8#077) == 0 )) || die 'Restore credentials require chmod 600'
|
||||
need "$DB_CLIENT_BIN"
|
||||
"$DB_CLIENT_BIN" --defaults-extra-file="${CNFS[i]}" --database="${TARGETS[i]}" --execute='SELECT 1' >/dev/null
|
||||
else
|
||||
check_source_path "${TARGETS[i]}"
|
||||
fi
|
||||
done
|
||||
if [[ $ENABLE_NEXTCLOUD == true && $ENABLE_NEXTCLOUD_MAINTENANCE != true ]]; then
|
||||
die 'Applying a restore with Nextcloud enabled requires maintenance-mode management'
|
||||
fi
|
||||
quiesce
|
||||
MUTATION_STARTED=true
|
||||
save_recovery
|
||||
for i in "${!SELECTED[@]}"; do
|
||||
label=${SELECTED[i]}
|
||||
source_path="$STAGING_DIR/extracted/$label/${SELECTED_ROOTS[i]}"
|
||||
if [[ ${IS_DB[i]} == true ]]; then
|
||||
[[ -f $source_path && ! -L $source_path && -s $source_path ]] || die 'SQL dump missing/empty'
|
||||
"$DB_CLIENT_BIN" --defaults-extra-file="${CNFS[i]}" --default-character-set=utf8mb4 --database="${TARGETS[i]}" < "$source_path"
|
||||
else
|
||||
exclusions "$label"
|
||||
if [[ $RESTORE_STRICT_DELETE == true ]]; then EXCLUDES+=(--delete-delay); fi
|
||||
rsync -aHAX --numeric-ids "${EXCLUDES[@]}" -- "$source_path/" "${TARGETS[i]}/"
|
||||
fi
|
||||
done
|
||||
# A backup captured maintenance=true. Enforce the recorded pre-restore state even
|
||||
# if the existing installation already was in maintenance mode.
|
||||
if [[ " ${SELECTED[*]} " == *' nextcloud '* || " ${SELECTED[*]} " == *' db-nextcloud '* ]]; then
|
||||
occ maintenance:mode --on
|
||||
occ maintenance:data-fingerprint
|
||||
fi
|
||||
resume_services || die 'Restore applied, but service recovery failed'
|
||||
MUTATION_STARTED=false
|
||||
echo 'SUCCESS: selected components restored. Perform application-level smoke tests.'
|
||||
Reference in new issue
Block a user