Make Nextcloud NAS backups independent, verified and retryable

scripts/install.sh

@@ -10,8 +10,17 @@ CONFIG_DIR="${XDG_CONFIG_HOME:-${HOME}/.config}/${APP_NAME}"
 STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/${APP_NAME}"
 SYSTEMD_USER_DIR="${XDG_CONFIG_HOME:-${HOME}/.config}/systemd/user"
 
-mkdir -p "$BIN_DIR" "$CONFIG_DIR" "$STATE_DIR/logs" "$SYSTEMD_USER_DIR"
+for required in python3 borg rclone findmnt flock timeout systemctl; do
+  command -v "$required" >/dev/null || { printf 'Missing dependency: %s\n' "$required" >&2; exit 1; }
+done
+[[ "$(borg --version)" == 'borg 1.'* ]] || { printf 'Borg 1.x required\n' >&2; exit 1; }
 
+EXISTING_CONFIG=false
+[[ ! -f "$CONFIG_DIR/config" ]] || EXISTING_CONFIG=true
+
+mkdir -p "${HOME}/.local/libexec" "$BIN_DIR" "$CONFIG_DIR" "$STATE_DIR/logs" "$SYSTEMD_USER_DIR"
+
+install -m 0750 "$ROOT_DIR/scripts/nextcloud-borg-runner.py" "${HOME}/.local/libexec/nextcloud-borg-runner.py"
 install -m 0750 "$ROOT_DIR/scripts/nextcloud-borg-backup" "$BIN_DIR/nextcloud-borg-backup"
 
 if [[ ! -f "$CONFIG_DIR/config" ]]; then
@@ -25,23 +34,38 @@ if [[ ! -f "$CONFIG_DIR/excludes" ]]; then
   install -m 0600 "$ROOT_DIR/etc/excludes.example" "$CONFIG_DIR/excludes"
 fi
 
-if [[ ! -f "$CONFIG_DIR/passphrase" ]]; then
-  if command -v openssl >/dev/null 2>&1; then
-    openssl rand -base64 48 > "$CONFIG_DIR/passphrase"
-  else
-    od -An -tx1 -N48 /dev/urandom | tr -d ' \n' > "$CONFIG_DIR/passphrase"
-    printf '\n' >> "$CONFIG_DIR/passphrase"
+# Respect installations that store the original key at a custom path.
+KEY_PATH="$(source "$CONFIG_DIR/config"; printf '%s' "${BORG_PASSPHRASE_FILE:-$CONFIG_DIR/passphrase}")"
+if [[ ! -f "$KEY_PATH" ]]; then
+  if [[ "$EXISTING_CONFIG" == true ]]; then
+    printf 'Existing installation but missing default passphrase. Restore the key or check the configured key path before updating.\n' >&2
+    exit 1
   fi
-  chmod 0600 "$CONFIG_DIR/passphrase"
-  printf 'Created Borg passphrase file: %s\n' "$CONFIG_DIR/passphrase"
+  # Never replace a missing key for an existing encrypted repository with random data.
+  if [[ -f "$CONFIG_DIR/config" ]]; then
+    source "$CONFIG_DIR/config"
+    if timeout 30 test -f "${REPO_DIR:-/net/thor/volume1/NetBackup/Nextcloud/borg-repo}/config"; then
+      printf 'Existing repository but missing passphrase; restore the original key first.\n' >&2
+      exit 1
+    fi
+  fi
+  if command -v openssl >/dev/null 2>&1; then
+    openssl rand -base64 48 > "$KEY_PATH"
+  else
+    od -An -tx1 -N48 /dev/urandom | tr -d ' \n' > "$KEY_PATH"
+    printf '\n' >> "$KEY_PATH"
+  fi
+  chmod 0600 "$KEY_PATH"
+  printf 'Created Borg passphrase file: %s\n' "$KEY_PATH"
   printf 'Store a copy of this passphrase somewhere safe for disaster recovery.\n'
 fi
 
-install -m 0644 "$ROOT_DIR/systemd/user/nextcloud-borg-backup.service" "$SYSTEMD_USER_DIR/nextcloud-borg-backup.service"
-install -m 0644 "$ROOT_DIR/systemd/user/nextcloud-borg-backup.timer" "$SYSTEMD_USER_DIR/nextcloud-borg-backup.timer"
-
+for unit in nextcloud-borg-backup.service nextcloud-borg-backup.timer nextcloud-borg-health.service nextcloud-borg-health.timer; do
+  install -m 0644 "$ROOT_DIR/systemd/user/$unit" "$SYSTEMD_USER_DIR/$unit"
+done
 systemctl --user daemon-reload
-systemctl --user enable --now nextcloud-borg-backup.timer
+systemctl --user enable nextcloud-borg-backup.timer nextcloud-borg-health.timer
+systemctl --user restart nextcloud-borg-backup.timer nextcloud-borg-health.timer
 
 printf '\nInstalled %s.\n' "$APP_NAME"
 printf 'Next steps:\n'

scripts/nextcloud-borg-backup

@@ -3,7 +3,7 @@ set -Eeuo pipefail
 umask 077
 
 APP_NAME="nextcloud-borg-backup"
-APP_VERSION="1.0.0"
+APP_VERSION="2.0.0"
 
 CONFIG_FILE="${NCB_CONFIG:-${XDG_CONFIG_HOME:-${HOME}/.config}/${APP_NAME}/config}"
 
@@ -30,10 +30,11 @@ shell_quote() {
 
 usage() {
   cat <<'EOF'
-nextcloud-borg-backup - Borg backup helper for a mounted Nextcloud folder
+nextcloud-borg-backup - Borg backup verified cloud-to-NAS backup and restore helper
 
 Usage:
-  nextcloud-borg-backup backup
+  nextcloud-borg-backup backup [--if-due]
+  nextcloud-borg-backup health [--notify]
   nextcloud-borg-backup init
   nextcloud-borg-backup list
   nextcloud-borg-backup info [ARCHIVE]
@@ -101,23 +102,6 @@ ensure_dirs() {
   mkdir -p "$LOG_DIR" "$STATE_DIR"
 }
 
-log_file_for_run() {
-  local timestamp
-  timestamp="$(date '+%Y-%m-%d_%H-%M-%S')"
-  printf '%s/%s_%s.log' "$LOG_DIR" "$APP_NAME" "$timestamp"
-}
-
-start_logging() {
-  ensure_dirs
-  local log_file
-  log_file="$(log_file_for_run)"
-  exec > >(tee -a "$log_file") 2>&1
-  info "== ${APP_NAME} ${APP_VERSION} =="
-  info "-- Time: $(date -Is)"
-  info "-- Config: $CONFIG_FILE"
-  info "-- Log: $log_file"
-}
-
 with_lock() {
   exec 9>"$LOCK_FILE"
   flock -n 9 || die "Another ${APP_NAME} run is already active: $LOCK_FILE"
@@ -141,7 +125,10 @@ borg_repo_exists() {
 init_repo() {
   require_command "$BORG_BIN"
   check_passphrase_for_encryption
-  mkdir -p "$(repo_parent)"
+  timeout 60 test -d "$(repo_parent)" || die "NAS parent unavailable; mount NAS before init"
+  local actual_source
+  actual_source="$(findmnt --direction backward --first-only -n -o SOURCE -T "$(repo_parent)")"
+  [[ "$actual_source" == "${EXPECTED_NAS_SOURCE:-thor:/volume1/NetBackup}" ]] || die "Wrong NAS backing mount: $actual_source"
   if borg_repo_exists; then
     info "-- Borg repository already exists: $REPO_DIR"
     return 0
@@ -150,77 +137,6 @@ init_repo() {
   "$BORG_BIN" init --encryption="$BORG_ENCRYPTION" "$REPO_DIR"
 }
 
-timeout_cmd() {
-  if have timeout; then
-    timeout "$PRECHECK_TIMEOUT_SECONDS" "$@"
-  else
-    "$@"
-  fi
-}
-
-count_source_entries() {
-  timeout_cmd find "$SOURCE_DIR" -mindepth 1 -maxdepth 1 -print -quit | wc -l
-}
-
-free_repo_gb() {
-  df -PBG "$(repo_parent)" | awk 'NR==2 { gsub(/G/, "", $4); print $4 }'
-}
-
-preflight() {
-  require_command "$BORG_BIN"
-  require_command flock
-  require_command df
-  require_command find
-
-  info "-- Source: $SOURCE_DIR"
-  info "-- Repository: $REPO_DIR"
-
-  timeout_cmd test -d "$SOURCE_DIR" || die "Source directory is not reachable: $SOURCE_DIR"
-  timeout_cmd test -r "$SOURCE_DIR" || die "Source directory is not readable: $SOURCE_DIR"
-
-  local entry_count
-  entry_count="$(count_source_entries | tr -d '[:space:]')"
-  if [[ "$ALLOW_EMPTY_SOURCE" != "true" && "${entry_count:-0}" -eq 0 ]]; then
-    die "Source directory looks empty. Refusing to create an empty backup."
-  fi
-
-  mkdir -p "$(repo_parent)"
-  timeout_cmd test -w "$(repo_parent)" || die "Repository parent is not writable/reachable: $(repo_parent)"
-
-  local free_gb
-  free_gb="$(free_repo_gb)"
-  info "-- Free space at repository parent: ${free_gb}G"
-  [[ "${free_gb:-0}" -ge "$MIN_FREE_REPO_GB" ]] || die "Not enough free space at repository parent; need ${MIN_FREE_REPO_GB}G"
-
-  if ! borg_repo_exists; then
-    init_repo
-  fi
-}
-
-create_archive() {
-  local timestamp archive
-  timestamp="$(date '+%Y-%m-%d_%H-%M-%S')"
-  archive="${ARCHIVE_PREFIX}-${timestamp}"
-
-  local borg_args
-  borg_args=(
-    create
-    --stats
-    --show-rc
-    --compression "$BORG_COMPRESSION"
-  )
-
-  [[ "$EXCLUDE_CACHES" == "true" ]] && borg_args+=(--exclude-caches)
-  [[ "$ONE_FILE_SYSTEM" == "true" ]] && borg_args+=(--one-file-system)
-  [[ -f "$EXCLUDE_FILE" ]] && borg_args+=(--exclude-from "$EXCLUDE_FILE")
-
-  info "-- Creating archive: $archive"
-  (
-    cd "$SOURCE_DIR"
-    "$BORG_BIN" "${borg_args[@]}" "${REPO_DIR}::${archive}" .
-  )
-}
-
 archive_names() {
   "$BORG_BIN" list --short --glob-archives "${ARCHIVE_PREFIX}-*" "$REPO_DIR"
 }
@@ -231,7 +147,7 @@ latest_archive() {
 
 prune_repo() {
   info "-- Pruning archives with retention: keep-within $RETENTION_WITHIN"
-  "$BORG_BIN" prune --list --show-rc --glob-archives "${ARCHIVE_PREFIX}-*" --keep-within "$RETENTION_WITHIN" "$REPO_DIR"
+  "$BORG_BIN" prune --list --show-rc --glob-archives "${ARCHIVE_PREFIX}-*" --keep-within "$RETENTION_WITHIN" --keep-last 7 "$REPO_DIR"
 }
 
 compact_repo() {
@@ -239,15 +155,6 @@ compact_repo() {
   "$BORG_BIN" compact "$REPO_DIR"
 }
 
-days_since_stamp() {
-  local stamp="$1"
-  [[ -f "$stamp" ]] || { printf '999999\n'; return 0; }
-  local now then
-  now="$(date +%s)"
-  then="$(stat -c %Y "$stamp")"
-  printf '%s\n' "$(((now - then) / 86400))"
-}
-
 check_repo() {
   local verify_data="${1:-false}"
   if [[ "$verify_data" == "true" ]]; then
@@ -259,30 +166,29 @@ check_repo() {
   fi
 }
 
-maybe_periodic_full_check() {
-  [[ "$FULL_CHECK_VERIFY_DATA" == "true" ]] || return 0
-  local stamp="${STATE_DIR}/last-full-check"
-  local age_days
-  age_days="$(days_since_stamp "$stamp")"
-  [[ "$age_days" -ge "$FULL_CHECK_INTERVAL_DAYS" ]] || return 0
-  check_repo true
-  date -Is > "$stamp"
+cmd_runner() {
+  load_config
+  export REPO_DIR ARCHIVE_PREFIX BORG_BIN BORG_COMPRESSION BORG_PASSPHRASE_FILE
+  export RETENTION_WITHIN PRUNE_AFTER_BACKUP COMPACT_AFTER_PRUNE
+  export STATE_DIR LOG_DIR MIN_FREE_REPO_GB EXCLUDE_FILE LOCK_FILE
+  export FULL_CHECK_INTERVAL_DAYS FULL_CHECK_VERIFY_DATA
+  # New settings are optional in existing configs, and inherited by the runner.
+  export REMOTE="${REMOTE:-Nextcloud:}"
+  export STAGING_DIR="${STAGING_DIR:-${HOME}/.local/share/nextcloud-borg-backup/staging}"
+  export EXPECTED_NAS_SOURCE="${EXPECTED_NAS_SOURCE:-thor:/volume1/NetBackup}"
+  export EXPECTED_NAS_FSTYPE="${EXPECTED_NAS_FSTYPE:-nfs4}"
+  export BACKUP_INTERVAL_HOURS="${BACKUP_INTERVAL_HOURS:-24}"
+  export MAX_BACKUP_AGE_HOURS="${MAX_BACKUP_AGE_HOURS:-36}"
+  export MIN_SOURCE_FILES="${MIN_SOURCE_FILES:-1}"
+  export MAX_SOURCE_DROP_PERCENT="${MAX_SOURCE_DROP_PERCENT:-20}"
+  export MAX_STAGE_DELETES="${MAX_STAGE_DELETES:-1000}"
+  export MIN_FREE_STAGE_GB="${MIN_FREE_STAGE_GB:-5}"
+  export COMMAND_TIMEOUT_SECONDS="${COMMAND_TIMEOUT_SECONDS:-21600}"
+  export NCB_RUNNER="${NCB_RUNNER:-${HOME}/.local/libexec/nextcloud-borg-runner.py}"
+  exec python3 "$NCB_RUNNER" "$@"
 }
 
-cmd_backup() {
-  load_config
-  start_logging
-  with_lock
-  preflight
-  create_archive
-  if [[ "$PRUNE_AFTER_BACKUP" == "true" ]]; then
-    prune_repo
-    [[ "$COMPACT_AFTER_PRUNE" == "true" ]] && compact_repo
-  fi
-  [[ "$REPOSITORY_CHECK_AFTER_BACKUP" == "true" ]] && check_repo false
-  maybe_periodic_full_check
-  info "-- Backup finished successfully"
-}
+cmd_backup() { cmd_runner backup "$@"; }
 
 cmd_list() {
   load_config
@@ -388,6 +294,8 @@ cmd_restore() {
 
 cmd_check() {
   load_config
+  ensure_dirs
+  with_lock
   require_command "$BORG_BIN"
   local verify_data=false
   [[ "${1:-}" == "--verify-data" ]] && verify_data=true
@@ -418,7 +326,8 @@ main() {
 
   case "$cmd" in
     backup) cmd_backup "$@" ;;
-    init) load_config; init_repo "$@" ;;
+    health) cmd_runner health "$@" ;;
+    init) load_config; ensure_dirs; with_lock; init_repo "$@" ;;
     list) cmd_list "$@" ;;
     info) cmd_info "$@" ;;
     search) cmd_search "$@" ;;
@@ -426,9 +335,9 @@ main() {
     umount|unmount) cmd_umount "$@" ;;
     restore) cmd_restore "$@" ;;
     check) cmd_check "$@" ;;
-    prune) load_config; prune_repo "$@"; compact_repo ;;
+    prune) load_config; ensure_dirs; with_lock; prune_repo "$@"; compact_repo ;;
     status) cmd_status "$@" ;;
-    break-lock|unlock) cmd_break_lock "$@" ;;
+    break-lock|unlock) load_config; ensure_dirs; with_lock; cmd_break_lock "$@" ;;
     -h|--help|help|"") usage ;;
     --version|version) printf '%s %s\n' "$APP_NAME" "$APP_VERSION" ;;
     *) usage >&2; die "Unknown command: $cmd" ;;

scripts/nextcloud-borg-runner.py

@@ -0,0 +1,392 @@
+#!/usr/bin/env python3
+"""Verified cloud staging and fail-closed Borg backup. Standard library only."""
+import datetime as dt
+import fcntl
+import hashlib
+import json
+import os
+import re
+from pathlib import Path, PurePosixPath
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+
+class BackupError(RuntimeError):
+    pass
+
+
+def log(message):
+    print(f'{dt.datetime.now().astimezone().isoformat(timespec="seconds")} {message}', flush=True)
+
+
+def atomic_json(path, data):
+    descriptor, name = tempfile.mkstemp(prefix=path.name + '.', dir=path.parent)
+    tmp = Path(name)
+    try:
+        with os.fdopen(descriptor, 'w') as stream:
+            json.dump(data, stream, ensure_ascii=False, indent=2)
+            stream.flush()
+            os.fsync(stream.fileno())
+        tmp.replace(path)
+        directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+        try:
+            os.fsync(directory)
+        finally:
+            os.close(directory)
+    finally:
+        tmp.unlink(missing_ok=True)
+
+
+def read_json(path):
+    if not path.exists():
+        return {}
+    return json.loads(path.read_text())
+
+
+class Runner:
+    def __init__(self):
+        e = os.environ
+        self.state = Path(e['STATE_DIR']).expanduser().absolute()
+        self.repo = Path(e['REPO_DIR']).absolute()
+        self.stage = Path(e['STAGING_DIR']).expanduser().absolute()
+        self.remote = e['REMOTE']
+        if not re.fullmatch(r'[A-Za-z0-9_.-]+', e['ARCHIVE_PREFIX']):
+            raise BackupError('Unsafe ARCHIVE_PREFIX')
+        self.borg = e.get('BORG_BIN', 'borg')
+        self.timeout = int(e.get('COMMAND_TIMEOUT_SECONDS', '21600'))
+        self.success_file = self.state / 'last-success.json'
+        self.attempt_file = self.state / 'last-attempt.json'
+        self.child = None
+        self.phase = 'preflight'
+        self.state.mkdir(parents=True, exist_ok=True, mode=0o700)
+        self.success = read_json(self.success_file)
+        # An old timestamp from a different repository/source must never suppress a backup.
+        if self.success and (self.success.get('repo') != str(self.repo) or
+                             self.success.get('remote') != self.remote):
+            self.success = {}
+
+    def run(self, args, *, capture=False, cwd=None, timeout=None):
+        log('Running: ' + ' '.join(map(str, args)))
+        self.child = subprocess.Popen(list(map(str, args)), cwd=cwd,
+                                      stdout=subprocess.PIPE if capture else None,
+                                      start_new_session=True, text=True)
+        try:
+            out, _ = self.child.communicate(timeout=timeout or self.timeout)
+            if self.child.returncode:
+                raise BackupError(f'{args[0]} failed (exit {self.child.returncode}); phase={self.phase}')
+            return out
+        except BaseException:
+            self.stop_child()
+            raise
+        finally:
+            self.child = None
+
+    def stop_child(self):
+        if self.child and self.child.poll() is None:
+            os.killpg(self.child.pid, signal.SIGTERM)
+            try:
+                self.child.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                os.killpg(self.child.pid, signal.SIGKILL)
+                self.child.wait()
+
+    def cloud(self, *args, capture=False):
+        return self.run(['rclone', *args, '--contimeout', '20s', '--timeout', '2m',
+                         '--retries', '3', '--low-level-retries', '10',
+                         '--log-level', 'INFO', '--stats', '1m'], capture=capture)
+
+    def nas(self):
+        # Trigger autofs, then verify the backing mount. Never mkdir or init on /net.
+        self.run(['timeout', '60', 'test', '-f', self.repo / 'config'], timeout=70)
+        mounts = json.loads(self.run(['findmnt', '--json', '--target', self.repo,
+                                    '--output', 'SOURCE,FSTYPE,MAJ:MIN'], capture=True, timeout=60))['filesystems']
+        device = self.repo.stat().st_dev
+        backing_device = f'{os.major(device)}:{os.minor(device)}'
+        mounts = [mount for mount in mounts if mount.get('maj:min') == backing_device]
+        expected = os.environ.get('EXPECTED_NAS_SOURCE', 'thor:/volume1/NetBackup')
+        fstype = os.environ.get('EXPECTED_NAS_FSTYPE', 'nfs4')
+        if len(mounts) != 1 or mounts[0]['source'] != expected or mounts[0]['fstype'] != fstype:
+            raise BackupError(f'Wrong NAS mount; expected {expected} ({fstype}), got {mounts}')
+        free = shutil.disk_usage(self.repo).free
+        if free < int(os.environ.get('MIN_FREE_REPO_GB', '5')) * 1024**3:
+            raise BackupError('Insufficient free NAS space')
+
+    def manifest(self):
+        raw = json.loads(self.cloud('lsjson', self.remote, '--recursive', '--files-only',
+                                    '--hash', '--no-mimetype', capture=True))
+        result = {}
+        for item in raw:
+            name = item['Path']
+            if (not name or name.startswith('/') or '..' in PurePosixPath(name).parts or
+                    '\x00' in name or item.get('IsDir') or item['Size'] < 0 or name in result):
+                raise BackupError('Unsafe, duplicate or unsupported cloud entry')
+            result[name] = {'size': item['Size'], 'mtime': item.get('ModTime'),
+                            'hashes': {k: v for k, v in item.get('Hashes', {}).items() if v}}
+        return result
+
+    def verify_local_hashes(self, manifest):
+        # Independent validation avoids any size-only fallback chosen inside rclone.
+        for name, item in manifest.items():
+            path = self.stage / name
+            if path.is_symlink() or not path.is_file() or path.stat().st_size != item['size']:
+                raise BackupError(f'Staging file missing, unsafe or wrong size: {name}')
+            hashes = {key.lower().replace('-', ''): value for key, value in item['hashes'].items()}
+            algorithm = next((key for key in ('sha1', 'md5') if hashes.get(key)), None)
+            if algorithm:
+                digest = hashlib.new(algorithm, usedforsecurity=False)
+                with path.open('rb') as stream:
+                    while chunk := stream.read(8 * 1024 * 1024):
+                        digest.update(chunk)
+                if digest.hexdigest().lower() != hashes[algorithm].lower():
+                    raise BackupError(f'Independent staging hash mismatch: {name}')
+        log('Independent local size/hash verification passed')
+
+    @staticmethod
+    def totals(manifest):
+        return len(manifest), sum(x['size'] for x in manifest.values())
+
+    def guard_source(self, manifest, baseline):
+        count, size = self.totals(manifest)
+        if count < int(os.environ.get('MIN_SOURCE_FILES', '1')) or size <= 0:
+            raise BackupError('Empty or implausibly small cloud source; refusing backup')
+        percent = float(os.environ.get('MAX_SOURCE_DROP_PERCENT', '20'))
+        if not 0 <= percent <= 100:
+            raise BackupError('MAX_SOURCE_DROP_PERCENT must be between 0 and 100')
+        for key, current in [('files', count), ('bytes', size)]:
+            previous = baseline.get(key, 0)
+            if previous and current < previous * (1 - percent / 100):
+                raise BackupError(f'Cloud {key} dropped by more than {percent}%: {previous} -> {current}')
+        log(f'Cloud inventory: {count} files, {size} bytes')
+        return count, size
+
+    def staging_preflight(self, manifest):
+        # Refuse symlinked, overlapping, mounted or foreign staging paths before rclone sync.
+        for other in (self.repo, self.state, Path.home() / 'Nextcloud'):
+            if self.stage == other or self.stage in other.parents or other in self.stage.parents:
+                raise BackupError(f'Unsafe staging overlap: {other}')
+        if self.stage.resolve() != self.stage:
+            raise BackupError('Staging path contains a symlink')
+        self.stage.mkdir(parents=True, exist_ok=True, mode=0o700)
+        os.chmod(self.stage, 0o700)
+        for base, dirs, files in os.walk(self.stage):
+            for name in dirs + files:
+                path = Path(base) / name
+                if path.is_symlink() or path.is_mount():
+                    raise BackupError(f'Unexpected symlink/mount in staging: {path}')
+        if self.stage.is_mount():
+            raise BackupError('Staging must be an ordinary local directory')
+        owner_file = self.stage.parent / (self.stage.name + '.owner.json')
+        expected = {'remote': self.remote, 'stage': str(self.stage)}
+        if owner_file.exists():
+            if read_json(owner_file) != expected:
+                raise BackupError('Staging belongs to another cloud source')
+        elif any(self.stage.iterdir()):
+            raise BackupError('Nonempty unowned staging directory; refusing sync')
+        else:
+            atomic_json(owner_file, expected)
+        mount = json.loads(self.run(['findmnt', '--json', '--target', self.stage,
+                                    '--output', 'FSTYPE'], capture=True))['filesystems'][0]
+        if mount['fstype'] not in ('ext4', 'btrfs', 'xfs', 'f2fs', 'zfs', 'tmpfs'):
+            raise BackupError(f'Staging must be local; found {mount["fstype"]}')
+        total = self.totals(manifest)[1]
+        # Conservative full-copy headroom also covers temporary transfer files.
+        need = total + int(os.environ.get('MIN_FREE_STAGE_GB', '5')) * 1024**3
+        if shutil.disk_usage(self.stage).free < need:
+            raise BackupError(f'Insufficient staging space: need {need} bytes free')
+
+    def baseline(self):
+        if self.success:
+            return self.success
+        data = json.loads(self.run([self.borg, 'info', '--json', '--last', '1',
+                                   '--glob-archives', os.environ['ARCHIVE_PREFIX'] + '-*',
+                                   self.repo], capture=True))
+        archives = data.get('archives', [])
+        if not archives:
+            return {}
+        stats = archives[0]['stats']
+        return {'files': stats['nfiles'], 'bytes': stats['original_size']}
+
+    def stale(self):
+        age = time.time() - self.success.get('completed_at', 0)
+        return age < 0 or age > float(os.environ.get('MAX_BACKUP_AGE_HOURS', '36')) * 3600
+
+    def notify(self, message):
+        log('WARNING: ' + message)
+        stamp = self.state / 'last-notification.json'
+        if time.time() - read_json(stamp).get('at', 0) < 6 * 3600:
+            return
+        if shutil.which('notify-send'):
+            result = subprocess.run(['notify-send', '--urgency=critical',
+                                     'Nextcloud NAS-Sicherung', message], timeout=15,
+                                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            if result.returncode == 0:
+                atomic_json(stamp, {'at': time.time()})
+
+    def backup(self, if_due=False):
+        with Path(os.environ['LOCK_FILE']).open('a') as lock:
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                log('Another backup/maintenance command holds the lock; skipping')
+                return
+            age = time.time() - self.success.get('completed_at', 0)
+            if if_due and 0 <= age < float(os.environ.get('BACKUP_INTERVAL_HOURS', '24')) * 3600:
+                if self.success.get('maintenance_pending') or read_json(self.attempt_file).get('phase') == 'retention':
+                    self.nas()
+                    self.maintenance()
+                    atomic_json(self.attempt_file, {'status': 'success', 'finished_at': time.time()})
+                log('Verified backup is still current; no transfer needed')
+                return
+            attempt = {'started_at': time.time(), 'status': 'running'}
+            atomic_json(self.attempt_file, attempt)
+            try:
+                self.perform()
+            except BaseException as exc:
+                atomic_json(self.attempt_file, {**attempt, 'finished_at': time.time(),
+                                              'status': 'failed', 'phase': self.phase, 'error': str(exc)})
+                try:
+                    self.notify('Sicherung fehlgeschlagen: ' + str(exc) +
+                                (' — Altersgrenze ohne bestätigtes Backup überschritten.' if self.stale() else ''))
+                except Exception as notify_error:
+                    log(f'Notification unavailable: {notify_error}')
+                raise
+            atomic_json(self.attempt_file, {**attempt, 'finished_at': time.time(), 'status': 'success'})
+
+    def check_identity(self):
+        identity = json.loads(self.run([self.borg, 'info', '--json', self.repo], capture=True))['repository']['id']
+        bound = read_json(self.state / 'repository.json')
+        if bound and bound != {'repo': str(self.repo), 'id': identity}:
+            raise BackupError('Repository identity changed; refusing backup/retention')
+        atomic_json(self.state / 'repository.json', {'repo': str(self.repo), 'id': identity})
+
+    def perform(self):
+        self.nas()
+        self.check_identity()
+        excludes = Path(os.environ['EXCLUDE_FILE'])
+        if excludes.exists() and any(x.strip() and not x.lstrip().startswith('#')
+                                     for x in excludes.read_text().splitlines()):
+            raise BackupError('Active legacy Borg excludes found; resolve before full cloud backup')
+        baseline = self.baseline()
+        self.phase = 'cloud inventory'
+        before = self.manifest()
+        count, size = self.guard_source(before, baseline)
+        self.staging_preflight(before)
+        self.phase = 'cloud sync'
+        common = {'md5', 'sha1'}
+        unhashed = sorted(name for name, item in before.items() if item['size'] and not any(
+            key.lower().replace('-', '') in common for key in item['hashes']))
+        # rclone's raw file lists cannot encode newline names. Fall back to a full byte read.
+        full_download = any('\n' in name or '\r' in name for name in unhashed)
+        file_list = self.state / 'unhashed-files.txt'
+        if unhashed and not full_download:
+            file_list.write_text('\n'.join(unhashed) + '\n')
+            log(f'Reading {len(unhashed)} files without server hashes in full')
+            self.cloud('copy', self.remote, str(self.stage), '--ignore-times',
+                       '--files-from-raw', str(file_list), '--transfers', '4')
+        sync_flags = ['--ignore-times'] if full_download else ['--checksum']
+        self.cloud('sync', self.remote, str(self.stage), '--check-first', '--delete-after',
+                   '--max-delete', os.environ.get('MAX_STAGE_DELETES', '1000'),
+                   '--create-empty-src-dirs', '--transfers', '4', '--checkers', '8', *sync_flags)
+        self.phase = 'cloud verification'
+        self.cloud('check', self.remote, str(self.stage), *(['--download'] if full_download else []))
+        if unhashed and not full_download:
+            self.cloud('check', self.remote, str(self.stage), '--download',
+                       '--files-from-raw', str(file_list))
+        self.verify_local_hashes(before)
+        after = self.manifest()
+        if before != after:
+            raise BackupError('Cloud changed during sync/verification; retry with a stable inventory')
+        atomic_json(self.state / 'verified-inventory.json', after)
+        self.nas()
+        self.phase = 'borg create'
+        archive = os.environ['ARCHIVE_PREFIX'] + '-' + dt.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+        target = f'{self.repo}::{archive}'
+        self.run([self.borg, 'create', '--stats', '--show-rc', '--compression',
+                  os.environ['BORG_COMPRESSION'], target, '.'], cwd=self.stage)
+        data = json.loads(self.run([self.borg, 'info', '--json', target], capture=True))
+        stats = data['archives'][0]['stats']
+        if stats['nfiles'] != count or stats['original_size'] != size:
+            raise BackupError(f'Archive inventory mismatch: {stats}; expected {count} files/{size} bytes')
+        self.phase = 'archive verification'
+        self.run([self.borg, 'check', '--archives-only', '--verify-data',
+                  '--glob-archives', archive, self.repo])
+        self.run([self.borg, 'check', '--repository-only', self.repo])
+        # Success means a complete verified archive. Retention failure must not lose this fact.
+        self.success = {'completed_at': time.time(), 'archive': archive, 'files': count,
+                        'bytes': size, 'repo': str(self.repo), 'remote': self.remote,
+                        'archive_id': data['archives'][0]['id'], 'maintenance_pending': True}
+        atomic_json(self.success_file, self.success)
+        log(f'BACKUP VERIFIED: {archive} ({count} files, {size} bytes)')
+        self.maintenance()
+        log('Backup and maintenance finished successfully')
+
+    def maintenance(self):
+        self.phase = 'retention'
+        self.check_identity()
+        if self.success.get('archive'):
+            archived = json.loads(self.run([self.borg, 'info', '--json',
+                                  f'{self.repo}::{self.success["archive"]}'], capture=True))['archives'][0]
+            if archived['id'] != self.success['archive_id']:
+                raise BackupError('Verified archive identity changed; refusing retention')
+        if os.environ.get('PRUNE_AFTER_BACKUP', 'true') == 'true':
+            self.run([self.borg, 'prune', '--list', '--show-rc', '--glob-archives',
+                      os.environ['ARCHIVE_PREFIX'] + '-*', '--keep-within',
+                      os.environ['RETENTION_WITHIN'], '--keep-last', '7', self.repo])
+            if os.environ.get('COMPACT_AFTER_PRUNE', 'true') == 'true':
+                self.run([self.borg, 'compact', self.repo])
+        if self.success:
+            self.success['maintenance_pending'] = False
+            atomic_json(self.success_file, self.success)
+        log('Retention finished successfully')
+
+
+def main():
+    os.umask(0o077)
+    runner = Runner()
+    def interrupted(signum, _frame):
+        runner.stop_child()
+        raise BackupError(f'Interrupted by signal {signum}')
+    signal.signal(signal.SIGTERM, interrupted)
+    signal.signal(signal.SIGINT, interrupted)
+    if sys.argv[1:] in (['health'], ['health', '--notify']):
+        print(json.dumps({'healthy': not runner.stale(), 'last_success': runner.success,
+                          'last_attempt': read_json(runner.attempt_file)}, ensure_ascii=False, indent=2))
+        if runner.stale() and '--notify' in sys.argv:
+            runner.notify('NAS-Sicherung überfällig: kein bestätigtes Backup innerhalb der Altersgrenze. Protokolle und health prüfen.')
+        return 2 if runner.stale() else 0
+    if sys.argv[1:] not in (['backup'], ['backup', '--if-due']):
+        raise BackupError('Usage: runner backup [--if-due] | health')
+    logs = Path(os.environ['LOG_DIR'])
+    logs.mkdir(parents=True, exist_ok=True, mode=0o700)
+    path = logs / ('backup-' + dt.datetime.now().strftime('%Y%m%d-%H%M%S') + '.log')
+    # Journal + file, without an orphaned tee process on timeout/interruption.
+    for old in logs.glob('backup-*.log'):
+        if old.is_file() and not old.is_symlink() and old.stat().st_mtime < time.time() - 90 * 86400:
+            old.unlink()
+    with path.open('a', buffering=1) as stream:
+        class Tee:
+            def write(self, data):
+                sys.__stdout__.write(data)
+                stream.write(data)
+            def flush(self):
+                sys.__stdout__.flush()
+                stream.flush()
+        sys.stdout = Tee()
+        # Child output goes to journal; lifecycle/phase/status additionally go to the run log.
+        try:
+            runner.backup('--if-due' in sys.argv)
+        finally:
+            sys.stdout = sys.__stdout__
+    return 0
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except (Exception, KeyboardInterrupt) as error:
+        log(f'ERROR: {error}')
+        sys.exit(1)

scripts/uninstall.sh

@@ -5,11 +5,13 @@ APP_NAME="nextcloud-borg-backup"
 BIN_DIR="${HOME}/.local/bin"
 SYSTEMD_USER_DIR="${XDG_CONFIG_HOME:-${HOME}/.config}/systemd/user"
 
-systemctl --user disable --now "${APP_NAME}.timer" 2>/dev/null || true
+systemctl --user stop "${APP_NAME}.service"
+systemctl --user disable --now "${APP_NAME}.timer" nextcloud-borg-health.timer 2>/dev/null || true
 systemctl --user daemon-reload
 
 rm -f "${SYSTEMD_USER_DIR}/${APP_NAME}.service" "${SYSTEMD_USER_DIR}/${APP_NAME}.timer"
-rm -f "${BIN_DIR}/${APP_NAME}"
+rm -f "${SYSTEMD_USER_DIR}/nextcloud-borg-health.service" "${SYSTEMD_USER_DIR}/nextcloud-borg-health.timer"
+rm -f "${BIN_DIR}/${APP_NAME}" "${HOME}/.local/libexec/nextcloud-borg-runner.py"
 systemctl --user daemon-reload
 
 printf 'Removed user service, timer, and executable.\n'