Added support for encyption and moved from one big backup file to smaller ones for each application.
This commit is contained in:
@@ -28,6 +28,15 @@ else
|
||||
fi
|
||||
|
||||
# ---------- Defaults (if not set in conf) ----------
|
||||
: "${ENABLE_MAIL:=false}"
|
||||
: "${MAIL_BACKUP_MODE:=live}" # live|stop (stop briefly stops postfix/dovecot)
|
||||
: "${MAIL_DIR:=/var/vmail}"
|
||||
: "${POSTFIX_DIR:=/etc/postfix}"
|
||||
: "${DOVECOT_DIR:=/etc/dovecot}"
|
||||
: "${ENABLE_MAIL_HOME_DIRS:=true}"
|
||||
: "${MAIL_HOME_DIRS:=/home/johannes /home/luciana}"
|
||||
: "${MAIL_HOME_EXCLUDES:=--exclude=OneDrive/ --exclude=.cache/ --exclude=.ccache/ --exclude=.local/share/Trash/}"
|
||||
|
||||
: "${WORKDIR:=/var/backups/app-backup}"
|
||||
: "${STAGING_ROOT:=${WORKDIR}/staging}"
|
||||
: "${ARCHIVE_DIR:=${WORKDIR}/archives}"
|
||||
@@ -47,7 +56,7 @@ fi
|
||||
: "${ENABLE_UPLOAD:=true}"
|
||||
|
||||
# large-file stability (OneDrive)
|
||||
: "${RCLONE_ONEDRIVE_CHUNK_SIZE:=64M}"
|
||||
: "${RCLONE_ONEDRIVE_CHUNK_SIZE:=80Mi}"
|
||||
: "${RCLONE_TIMEOUT:=1h}"
|
||||
: "${RCLONE_CONTIMEOUT:=30s}"
|
||||
: "${RCLONE_TRANSFERS:=2}"
|
||||
@@ -80,9 +89,25 @@ RCLONE_OUTPUT_FILE=""
|
||||
SIZES_FILE=""
|
||||
|
||||
# ---------- Helpers ----------
|
||||
die() { echo "ERROR: $*"; exit 1; }
|
||||
die() { STATUS="FAIL"; ERROR_SUMMARY="${1:-Unknown error}"; echo "ERROR: $*"; exit 1; }
|
||||
have() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
require_crypt_remote() {
|
||||
local remote_name="$1"
|
||||
local cfg
|
||||
cfg="$("$RCLONE_BIN" config show "$remote_name" 2>/dev/null || true)"
|
||||
[[ -n "$cfg" ]] || die "rclone remote \"$remote_name\" not found in config"
|
||||
echo "$cfg" | grep -Eq "^type\s*=\s*crypt\s*$" || die "Remote \"$remote_name\" is not a crypt remote (type!=crypt). Refusing to upload unencrypted."
|
||||
# These options may be omitted if defaults are used. If present and set to disable encryption, fail.
|
||||
if echo "$cfg" | grep -Eq "^filename_encryption\s*=\s*off\s*$"; then
|
||||
die "Remote \"$remote_name\" has filename_encryption=off. Filenames would be visible in OneDrive."
|
||||
fi
|
||||
if echo "$cfg" | grep -Eq "^directory_name_encryption\s*=\s*(false|off)\s*$"; then
|
||||
die "Remote \"$remote_name\" has directory_name_encryption disabled. Directory names would be visible in OneDrive."
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
human_bytes() { local b="${1:-0}"; if have numfmt; then numfmt --to=iec-i --suffix=B "$b"; else echo "${b}B"; fi; }
|
||||
bytes_of_path() { local p="$1"; [[ -e "$p" ]] && (du -sb "$p" 2>/dev/null | awk '{print $1}' || du -sB1 "$p" | awk '{print $1}') || echo 0; }
|
||||
free_bytes_workdir_fs() { df -PB1 "$WORKDIR" | awk 'NR==2{print $4}'; }
|
||||
@@ -321,9 +346,72 @@ if [[ "${ENABLE_GITEA:-false}" == "true" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
\
|
||||
if [[ "${ENABLE_MAIL}" == "true" ]]; then
|
||||
echo "-- Mail files..."
|
||||
mkdir -p "$STAGING_DIR/files/mail" "$STAGING_DIR/files/postfix" "$STAGING_DIR/files/dovecot" "$STAGING_DIR/files/mail-home"
|
||||
|
||||
# Optional short stop for maximum consistency
|
||||
if [[ "${MAIL_BACKUP_MODE}" == "stop" ]]; then
|
||||
echo "-- Stopping postfix/dovecot for mail backup..."
|
||||
systemctl stop postfix || true
|
||||
systemctl stop dovecot || true
|
||||
fi
|
||||
|
||||
[[ -n "${MAIL_DIR:-}" && -d "${MAIL_DIR}" ]] && rsync_dir "${MAIL_DIR}" "$STAGING_DIR/files/mail" || true
|
||||
[[ -n "${POSTFIX_DIR:-}" && -d "${POSTFIX_DIR}" ]] && rsync_dir "${POSTFIX_DIR}" "$STAGING_DIR/files/postfix" || true
|
||||
[[ -n "${DOVECOT_DIR:-}" && -d "${DOVECOT_DIR}" ]] && rsync_dir "${DOVECOT_DIR}" "$STAGING_DIR/files/dovecot" || true
|
||||
|
||||
# Home mail dirs (per-user mail files under /home)
|
||||
if [[ "${ENABLE_MAIL_HOME_DIRS}" == "true" ]]; then
|
||||
for h in ${MAIL_HOME_DIRS}; do
|
||||
if [[ -d "${h}" ]]; then
|
||||
user="$(basename "${h}")"
|
||||
echo "-- Mail (home): ${h}"
|
||||
dest="$STAGING_DIR/files/mail-home/${user}"
|
||||
mkdir -p "$dest"
|
||||
|
||||
# Some home subdirs (e.g. FUSE mounts like OneDrive) may deny access or cause
|
||||
# noise during backup. We keep this resilient:
|
||||
# - skip if the user does not exist
|
||||
# - run rsync as root (so we can write into the root-owned staging dir)
|
||||
# - prevent crossing filesystem boundaries (skip mounts)
|
||||
# - exclude noisy/unwanted dirs via MAIL_HOME_EXCLUDES
|
||||
if ! id "${user}" >/dev/null 2>&1; then
|
||||
echo "WARNING: mail-home user '${user}' does not exist (skipping ${h})" >&2
|
||||
continue
|
||||
fi
|
||||
|
||||
chown root:root "$dest" 2>/dev/null || true
|
||||
chmod 0750 "$dest" 2>/dev/null || true
|
||||
|
||||
set +e
|
||||
rsync -aHAX --numeric-ids --info=stats2 --one-file-system ${MAIL_HOME_EXCLUDES} "${h}/" "${dest}/"
|
||||
rc=$?
|
||||
set -e
|
||||
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
echo "WARNING: rsync for ${h} returned code ${rc} (continuing)" >&2
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "${MAIL_BACKUP_MODE}" == "stop" ]]; then
|
||||
echo "-- Starting postfix/dovecot after mail backup..."
|
||||
systemctl start dovecot || true
|
||||
systemctl start postfix || true
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# ---------- Size summary ----------
|
||||
SIZES_FILE="${STAGING_DIR}/meta/sizes.txt"
|
||||
{
|
||||
echo "Mail-home staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/mail-home")")"
|
||||
echo "Mail staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/mail")")"
|
||||
echo "Postfix staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/postfix")")"
|
||||
echo "Dovecot staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/dovecot")")"
|
||||
echo "DB dumps staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/db")")"
|
||||
echo "WordPress staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/wordpress")")"
|
||||
echo "Nextcloud code staged: $(human_bytes "$(bytes_of_path "$STAGING_DIR/files/nextcloud")")"
|
||||
@@ -340,8 +428,7 @@ make_archive() {
|
||||
local label="$1" src_rel="$2"
|
||||
local tar_file="${ARCHIVE_DIR}/${ARCHIVE_PREFIX}_${ts}_${label}.tar"
|
||||
local out_file
|
||||
|
||||
echo "-- Creating archive (${label}): ${tar_file}"
|
||||
echo "-- Creating archive (${label}): ${tar_file}" >&2
|
||||
(
|
||||
cd "$STAGING_DIR"
|
||||
tar --numeric-owner --xattrs --acls -cf "$tar_file" "$src_rel"
|
||||
@@ -349,16 +436,16 @@ make_archive() {
|
||||
|
||||
if [[ "$COMPRESSOR" == "zstd" ]]; then
|
||||
out_file="${tar_file}.zst"
|
||||
echo "-- Compressing (zstd): ${out_file}"
|
||||
ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" zstd -T0 -19 --rm "$tar_file"
|
||||
zstd -t "$out_file"
|
||||
echo "-- Compressing (zstd): ${out_file}" >&2
|
||||
ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" zstd -T0 -19 --rm -q "$tar_file" 1>&2
|
||||
zstd -t "$out_file" 1>&2
|
||||
else
|
||||
out_file="${tar_file}.gz"
|
||||
echo "-- Compressing (gzip): ${out_file}"
|
||||
echo "-- Compressing (gzip): ${out_file}" >&2
|
||||
ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" gzip -9 "$tar_file"
|
||||
gzip -t "$out_file"
|
||||
fi
|
||||
|
||||
echo "$out_file" >&2
|
||||
echo "$out_file"
|
||||
}
|
||||
|
||||
@@ -380,6 +467,17 @@ if [[ "${ENABLE_GITEA:-false}" == "true" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Filter empty archive entries
|
||||
ARCHIVES=($(for a in "${ARCHIVES[@]:-}"; do [[ -n "$a" ]] && echo "$a"; done))
|
||||
|
||||
# Mail archives (vmail/postfix/dovecot + optional /home mail dirs)
|
||||
if [[ "${ENABLE_MAIL}" == "true" ]]; then
|
||||
ARCHIVES+=("$(make_archive "mail" "files/mail")")
|
||||
ARCHIVES+=("$(make_archive "mail-home" "files/mail-home")")
|
||||
ARCHIVES+=("$(make_archive "postfix" "files/postfix")")
|
||||
ARCHIVES+=("$(make_archive "dovecot" "files/dovecot")")
|
||||
fi
|
||||
|
||||
echo "-- Archives created:"
|
||||
for f in "${ARCHIVES[@]}"; do
|
||||
echo " - $f ($(du -h "$f" | awk '{print $1}'))"
|
||||
@@ -394,8 +492,20 @@ if [[ "${ENABLE_UPLOAD}" == "true" ]]; then
|
||||
RCLONE_STATUS="RUNNING"
|
||||
remote_run="${RCLONE_REMOTE_BASE}/${ARCHIVE_PREFIX}_${ts}"
|
||||
|
||||
echo "-- rclone remote check: ${RCLONE_REMOTE_BASE}"
|
||||
"$RCLONE_BIN" lsf "${RCLONE_REMOTE_BASE}" --max-depth 1 >/dev/null 2>&1 || die "Remote not reachable: ${RCLONE_REMOTE_BASE}"
|
||||
# Split "Remote:path" into remote name + path. Some rclone subcommands require only the remote name.
|
||||
[[ "${RCLONE_REMOTE_BASE}" == *:* ]] || die "RCLONE_REMOTE_BASE must be in the form Remote:path (got: ${RCLONE_REMOTE_BASE})"
|
||||
remote_name="${RCLONE_REMOTE_BASE%%:*}"
|
||||
remote_base_path="${RCLONE_REMOTE_BASE#*:}"
|
||||
|
||||
echo "-- rclone remote check (crypt enforced): ${remote_name}"
|
||||
require_crypt_remote "${remote_name}"
|
||||
|
||||
# Check remote root reachability (works even if base folder doesn't exist yet)
|
||||
"$RCLONE_BIN" lsd "${remote_name}:" --max-depth 1 >/dev/null 2>&1 || die "Remote not reachable: ${remote_name}:"
|
||||
|
||||
# Ensure base folder exists (best effort)
|
||||
echo "-- Ensuring remote base folder exists: ${RCLONE_REMOTE_BASE}"
|
||||
"$RCLONE_BIN" mkdir "${RCLONE_REMOTE_BASE}" >/dev/null 2>&1 || true
|
||||
|
||||
echo "-- Creating remote folder: ${remote_run}"
|
||||
"$RCLONE_BIN" mkdir "${remote_run}" >/dev/null 2>&1 || true
|
||||
@@ -418,6 +528,7 @@ if [[ "${ENABLE_UPLOAD}" == "true" ]]; then
|
||||
|
||||
echo "-- Uploading archives to: ${remote_run} (log: ${RCLONE_OUTPUT_FILE})"
|
||||
for f in "${ARCHIVES[@]}"; do
|
||||
[[ -n "$f" ]] || continue
|
||||
echo "-- Upload: $(basename "$f")"
|
||||
if ionice -c "${IONICE_CLASS}" -n "${IONICE_LEVEL}" nice -n "${NICE_LEVEL}" "$RCLONE_BIN" copy "$f" "${remote_run}" "${common_args[@]}" | tee -a "$RCLONE_OUTPUT_FILE"
|
||||
then
|
||||
@@ -429,6 +540,10 @@ if [[ "${ENABLE_UPLOAD}" == "true" ]]; then
|
||||
done
|
||||
RCLONE_STATUS="OK"
|
||||
|
||||
# Show what landed on the remote for this run (small folder: archives only)
|
||||
echo "-- Remote listing (this run): ${remote_run}"
|
||||
"$RCLONE_BIN" lsl "${remote_run}" --max-depth 1 --log-level INFO || echo "WARNING: Could not list remote run folder: ${remote_run}"
|
||||
|
||||
if [[ "${ENABLE_REMOTE_RETENTION}" == "true" ]]; then
|
||||
echo "-- Remote retention: delete objects older than ${REMOTE_RETENTION_DAYS}d (best effort)"
|
||||
"$RCLONE_BIN" delete "${RCLONE_REMOTE_BASE}" --min-age "${REMOTE_RETENTION_DAYS}d" --log-level INFO || true
|
||||
|
||||
Reference in New Issue
Block a user