Restore detailed backup mail reports with bounded local delivery
This commit is contained in:
admin committed
1 parent
3374019ad6
commit
99da6460d2
10 files changed
+396
-13
No files matched your search
+28
-4
@@ -35,6 +35,9 @@ load_config() {
|
||||
: "${RCLONE_STATS_INTERVAL:=1m}"
|
||||
: "${RCLONE_TRANSFERS:=2}" "${RCLONE_CHECKERS:=4}" "${RCLONE_ONEDRIVE_CHUNK_SIZE:=80Mi}"
|
||||
: "${RCLONE_BWLIMIT:=0}" "${ENABLE_MAIL_REPORT:=true}" "${MAIL_TO:=johannes}"
|
||||
# Keep the old report settings; an empty sender uses app-backup@<FQDN>.
|
||||
: "${MAIL_FROM:=}" "${MAIL_SUBJECT_PREFIX:=[app-backup]}" "${MAIL_INCLUDE_LOG_TAIL_LINES:=200}"
|
||||
: "${MAIL_SENDMAIL_BIN:=/usr/sbin/sendmail}" "${MAIL_SEND_TIMEOUT:=30}"
|
||||
: "${ENABLE_WORDPRESS:=false}" "${ENABLE_NEXTCLOUD:=false}" "${ENABLE_NEXTCLOUD_DATA:=true}"
|
||||
: "${ENABLE_GITEA:=false}" "${ENABLE_MAIL:=false}" "${ENABLE_DB_DUMPS:=true}"
|
||||
: "${ENABLE_NEXTCLOUD_MAINTENANCE:=true}" "${NC_OCC_USER:=apache}"
|
||||
@@ -116,10 +119,16 @@ init_runtime() {
|
||||
done
|
||||
RUN_ID="${ARCHIVE_PREFIX}_$(date -u +%Y-%m-%d_%H-%M-%S)_$(python3 -c 'import secrets; print(secrets.token_hex(4))')"
|
||||
LOG_FILE="$LOG_DIR/${OPERATION}_${RUN_ID}.log"
|
||||
RUN_STARTED_EPOCH=$(date +%s)
|
||||
BACKUP_PHASE=preflight
|
||||
UPLOAD_STATUS=NOT_STARTED
|
||||
if [[ $ENABLE_UPLOAD != true ]]; then UPLOAD_STATUS=DISABLED; fi
|
||||
# Journald already captures service stdout. No dependency on systemd-cat in terminals.
|
||||
# Keep the log pipe alive while systemd sends TERM to the whole control group.
|
||||
# tee exits on EOF after the recovery traps close their writers.
|
||||
exec {REPORT_STDOUT}>&1 {REPORT_STDERR}>&2
|
||||
exec > >(trap '' INT TERM; exec tee -a "$LOG_FILE") 2>&1
|
||||
REPORT_LOGGER_PID=$!
|
||||
echo "== $OPERATION $RUN_ID =="
|
||||
local security_context
|
||||
if security_context=$(id -Z 2>/dev/null); then echo "-- SELinux process context: $security_context"; fi
|
||||
@@ -484,17 +493,32 @@ finish() {
|
||||
else
|
||||
resume_services || code=1
|
||||
fi
|
||||
local report_sizes="$LOG_DIR/report_$RUN_ID.sizes.json"
|
||||
if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
|
||||
# Collect after service recovery but before successful staging cleanup.
|
||||
python3 "$LIB_DIR/report.py" snapshot "$report_sizes" "${STAGING_DIR:-}" "${RUN_DIR:-${PARTIAL_DIR:-}}" ||
|
||||
echo 'WARNING: Report sizes unavailable'
|
||||
fi
|
||||
if [[ $code == 0 && -n ${STAGING_DIR:-} ]]; then rm -rf -- "$STAGING_DIR" || code=1; fi
|
||||
if [[ $code != 0 ]]; then echo "FAILED (exit $code). Retained work: ${STAGING_DIR:-none}"; fi
|
||||
# Close the logger's writers and wait for EOF so the mail tail includes the
|
||||
# final recovery/status messages. Reporting itself uses the original stdout.
|
||||
exec 1>&"$REPORT_STDOUT" 2>&"$REPORT_STDERR"
|
||||
exec {REPORT_STDOUT}>&- {REPORT_STDERR}>&-
|
||||
wait "$REPORT_LOGGER_PID" || code=1
|
||||
# Persist status separately from --check; failed status writes also fail the job.
|
||||
if [[ ${CHECK:-false} != true ]]; then
|
||||
helper status "$WORKDIR" "$OPERATION" "$RUN_ID" "$code" || code=1
|
||||
fi
|
||||
if [[ ${CHECK:-false} != true && $ENABLE_MAIL_REPORT == true && $OPERATION == backup ]]; then
|
||||
if [[ -x /usr/sbin/sendmail && $MAIL_TO != *$'\n'* && $MAIL_TO != *$'\r'* ]]; then
|
||||
{ printf 'To: %s\nSubject: [app-backup] exit=%s %s\nContent-Type: text/plain; charset=UTF-8\n\nLog: %s\n' "$MAIL_TO" "$code" "$RUN_ID" "$LOG_FILE"; tail -n 80 "$LOG_FILE"; } |
|
||||
/usr/sbin/sendmail -t || echo "WARNING: report delivery failed"
|
||||
else echo "WARNING: mail report unavailable (sendmail/recipient)"; fi
|
||||
python3 "$LIB_DIR/report.py" deliver --sizes "$report_sizes" --log "$LOG_FILE" \
|
||||
--output "$LOG_DIR/report_$RUN_ID.eml" --to "$MAIL_TO" --sender "$MAIL_FROM" \
|
||||
--subject-prefix "$MAIL_SUBJECT_PREFIX" --run "$RUN_ID" --exit-code "$code" \
|
||||
--started "$RUN_STARTED_EPOCH" --phase "$BACKUP_PHASE" --upload-status "$UPLOAD_STATUS" \
|
||||
--config "$CONFIG_FILE" --workdir "$WORKDIR" --archive-dir "$ARCHIVE_DIR" \
|
||||
--compression "$COMPRESSOR" --remote "$RCLONE_REMOTE_BASE" --sendmail "$MAIL_SENDMAIL_BIN" \
|
||||
--tail-lines "$MAIL_INCLUDE_LOG_TAIL_LINES" --timeout "$MAIL_SEND_TIMEOUT" 2>&1 | tee -a "$LOG_FILE"
|
||||
rm -f -- "$report_sizes"
|
||||
fi
|
||||
exit "$code"
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build private UTF-8 backup reports; never read executable config/credentials.
|
||||
|
||||
Sizes are captured before staging cleanup. Delivery uses only the local MTA and
|
||||
has a timeout; a reporting failure never turns a failed backup into a success.
|
||||
"""
|
||||
import argparse
|
||||
from collections import deque
|
||||
import datetime as dt
|
||||
from email.message import EmailMessage
|
||||
from email.policy import SMTP
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def allocated(path):
|
||||
"""Report disk allocation, like du in the old report, without following links."""
|
||||
result = subprocess.run(['du', '-s', '-B1', '--', str(path)],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return int(result.stdout.split()[0])
|
||||
|
||||
|
||||
def snapshot(output, staging, archives):
|
||||
data = {'staged': {}, 'archives': {}}
|
||||
if staging and Path(staging).is_dir():
|
||||
root = Path(staging)
|
||||
# Measure components independently; the total accounts for shared links.
|
||||
paths = sorted((root/'files').glob('*')) + [root/'db', root/'meta']
|
||||
for path in paths:
|
||||
if path.exists():
|
||||
data['staged'][path.name] = allocated(path)
|
||||
data['staging_total'] = allocated(root)
|
||||
if archives and Path(archives).is_dir():
|
||||
for path in sorted(Path(archives).glob('*.tar.*')):
|
||||
if path.is_file() and not path.is_symlink():
|
||||
data['archives'][path.name] = path.stat().st_size
|
||||
Path(output).write_text(json.dumps(data), encoding='utf-8')
|
||||
|
||||
|
||||
def human(value):
|
||||
for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
|
||||
if value < 1024 or unit == 'TiB':
|
||||
return f'{value:.1f} {unit}'
|
||||
value /= 1024
|
||||
|
||||
|
||||
def deliver(args):
|
||||
host = socket.getfqdn()
|
||||
now = dt.datetime.now().astimezone()
|
||||
state = 'SUCCESS' if args.exit_code == 0 else 'FAIL'
|
||||
try:
|
||||
sizes = json.loads(Path(args.sizes).read_text())
|
||||
except (OSError, ValueError):
|
||||
sizes = {}
|
||||
# Bounded memory even for long-running rclone logs. Tail length is configurable.
|
||||
with open(args.log, encoding='utf-8', errors='replace') as stream:
|
||||
tail = ''.join(deque(stream, maxlen=args.tail_lines))
|
||||
upload = {'NOT_STARTED': 'Nicht gestartet', 'DISABLED': 'Deaktiviert (nur lokal)',
|
||||
'COPY': 'Nicht abgeschlossen: Upload', 'CHECK': 'Nicht abgeschlossen: Rückleseprüfung',
|
||||
'MARKER': 'Nicht abgeschlossen: Abschlussmarker',
|
||||
'VERIFIED': 'OK (Inhalt und Abschlussmarker zurückgelesen)'}[args.upload_status]
|
||||
body = [
|
||||
'Backup Report', '=============', f'Zeit: {now.isoformat(timespec="seconds")}',
|
||||
f'Host: {host}', f'Status: {state}', f'Exit code: {args.exit_code}',
|
||||
f'Dauer: {max(0, int(time.time()) - args.started)}s', f'Lauf: {args.run}',
|
||||
f'Letzte Phase: {args.phase}', '', f'Config: {args.config}', f'Log: {args.log}',
|
||||
f'Workdir: {args.workdir}', f'Archive dir: {args.archive_dir}',
|
||||
f'Kompression: {args.compression}', '', 'Remote', '------',
|
||||
f'Upload: {str(args.upload_status != "DISABLED").lower()}',
|
||||
f'Remote base: {args.remote or "nicht konfiguriert"}', f'Upload Status: {upload}',
|
||||
'Die separate Update-Abnahme steht in result.txt; dies ist der Backup-Bericht.',
|
||||
'', 'Größen', '------', 'Staging: belegter Speicher vor Bereinigung; Archive: Dateigrößen.',
|
||||
]
|
||||
for label, value in sizes.get('staged', {}).items():
|
||||
body.append(f'{label} staged: {human(value)}')
|
||||
body.append('Staging total: '+(human(sizes['staging_total']) if 'staging_total' in sizes else 'nicht verfügbar'))
|
||||
body.extend(['', 'Archive', '-------'])
|
||||
for name, value in sizes.get('archives', {}).items():
|
||||
body.append(f'{name}: {human(value)} ({value} Bytes)')
|
||||
if not sizes.get('archives'):
|
||||
body.append('Keine Archive erfasst.')
|
||||
else:
|
||||
body.append('Archive total: '+human(sum(sizes['archives'].values())))
|
||||
body.extend(['', f'Log-Auszug (letzte {args.tail_lines} Zeilen)', '-----------------', tail])
|
||||
message = EmailMessage(policy=SMTP)
|
||||
# EmailMessage rejects CR/LF in headers and encodes non-ASCII safely.
|
||||
sender = args.sender or f'app-backup@{host}'
|
||||
if not args.to.strip() or not sender.strip():
|
||||
raise ValueError('Mail recipient and sender must not be empty')
|
||||
message['From'] = sender
|
||||
message['To'] = args.to
|
||||
message['Subject'] = f'{args.subject_prefix} {state} {host} {args.run}'
|
||||
message['Date'] = now
|
||||
message.set_content('\n'.join(body), charset='utf-8')
|
||||
raw = message.as_bytes()
|
||||
Path(args.output).write_bytes(raw)
|
||||
# -oi prevents a single dot in a log line from terminating the message.
|
||||
# Headers supply recipients; no shell or SMTP credentials are involved.
|
||||
subprocess.run([args.sendmail, '-oi', '-t'], input=raw, check=True, timeout=args.timeout)
|
||||
print(f'Mail report handed to local MTA: {args.output}')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest='command', required=True)
|
||||
snap = commands.add_parser('snapshot')
|
||||
for name in ('output', 'staging', 'archives'):
|
||||
snap.add_argument(name)
|
||||
mail = commands.add_parser('deliver')
|
||||
for name in ('sizes', 'log', 'output', 'to', 'sender', 'subject-prefix', 'run', 'phase',
|
||||
'upload-status', 'config', 'workdir', 'archive-dir', 'compression', 'remote', 'sendmail'):
|
||||
mail.add_argument('--'+name, required=True)
|
||||
for name in ('exit-code', 'started', 'tail-lines', 'timeout'):
|
||||
mail.add_argument('--'+name, type=int, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.command == 'snapshot':
|
||||
snapshot(args.output, args.staging, args.archives)
|
||||
else:
|
||||
if not 0 <= args.tail_lines <= 10000 or not 1 <= args.timeout <= 300:
|
||||
raise ValueError('Invalid mail tail length or timeout')
|
||||
deliver(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
os.umask(0o077)
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
# Do not echo header values or MTA arguments which might include addresses.
|
||||
print(f'WARNING: mail report failed ({type(exc).__name__}); inspect local report/MTA.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in new issue
Block a user