Guard writer units against activation and fix backup cancellation
This commit is contained in:
admin committed
1 parent
c0a24b63fc
commit
ff152c7f0f
9 files changed
+372
-16
No files matched your search
+99
-7
@@ -116,12 +116,14 @@ init_runtime() {
|
||||
RUN_ID="${ARCHIVE_PREFIX}_$(date -u +%Y-%m-%d_%H-%M-%S)_$(python3 -c 'import secrets; print(secrets.token_hex(4))')"
|
||||
LOG_FILE="$LOG_DIR/${OPERATION}_${RUN_ID}.log"
|
||||
# Journald already captures service stdout. No dependency on systemd-cat in terminals.
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
# Keep the log pipe alive while systemd sends TERM to the whole control group.
|
||||
# tee exits on EOF after the recovery traps close their writers.
|
||||
exec > >(trap '' INT TERM; exec tee -a "$LOG_FILE") 2>&1
|
||||
echo "== $OPERATION $RUN_ID =="
|
||||
local security_context
|
||||
if security_context=$(id -Z 2>/dev/null); then echo "-- SELinux process context: $security_context"; fi
|
||||
need rsync; need tar; need "$COMPRESSOR"
|
||||
STOPPED_SERVICES=(); NC_CHANGED=false; NC_WAS_ON=false; MUTATION_STARTED=false
|
||||
STOPPED_SERVICES=(); GUARDED_SERVICES=(); GUARD_FILES=(); GUARD_STAMPS=(); SERVICE_GUARD_DIR=''; NC_CHANGED=false; NC_WAS_ON=false; MUTATION_STARTED=false
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
trap finish EXIT
|
||||
@@ -284,8 +286,83 @@ exclusions() {
|
||||
fi
|
||||
}
|
||||
|
||||
# A stop alone does not exclude timer/monitor/dependency activation. Attach a
|
||||
# run-owned runtime condition before stopping each writer. Unlike a runtime mask,
|
||||
# a drop-in also works for administrator units whose main file lives in /etc.
|
||||
# This runtime root is overridable only by the trusted config (isolated tests).
|
||||
install_service_guard() {
|
||||
local unit=$1 id load dir file paths existing
|
||||
for id in "${GUARDED_SERVICES[@]}"; do [[ $id != "$unit" ]] || return 0; done
|
||||
load=$(systemctl show "$unit" --property=LoadState --value)
|
||||
[[ $load == loaded || $load == masked ]] || die "Cannot guard unloaded unit: $unit ($load)"
|
||||
if [[ $load == masked ]]; then
|
||||
# Preserve pre-existing masks verbatim; never claim ownership of them.
|
||||
GUARDED_SERVICES+=("$unit"); GUARD_FILES+=(''); return 0
|
||||
fi
|
||||
id=$(systemctl show "$unit" --property=Id --value)
|
||||
[[ $id =~ ^[a-zA-Z0-9_@:.\\-]+\.(service|timer|socket|path)$ ]] || die "Unsupported writer unit name: $id"
|
||||
: "${SERVICE_GUARD_RUNTIME_ROOT:=/run}"
|
||||
[[ $SERVICE_GUARD_RUNTIME_ROOT =~ ^/[a-zA-Z0-9_./-]+$ ]] || die 'Unsafe service guard runtime root'
|
||||
if [[ -z $SERVICE_GUARD_DIR ]]; then
|
||||
SERVICE_GUARD_DIR=$(mktemp -d "$SERVICE_GUARD_RUNTIME_ROOT/app-backup-guard-XXXXXXXX")
|
||||
fi
|
||||
dir="$SERVICE_GUARD_RUNTIME_ROOT/systemd/system/$id.d"
|
||||
[[ ! -L $dir ]] || die "Symlink writer drop-in directory refused: $dir"
|
||||
mkdir -p -- "$dir"
|
||||
[[ $(stat -c %u "$dir") == 0 ]] || die "Writer drop-in directory must belong to root: $dir"
|
||||
(( (8#$(stat -c %a "$dir") & 8#022) == 0 )) || die "Unsafe writer drop-in directory: $dir"
|
||||
file="$dir/zzzz-app-backup-$RUN_ID.conf"
|
||||
for existing in "${GUARD_FILES[@]}"; do [[ $existing != "$file" ]] || return 0; done
|
||||
[[ ! -e $file && ! -L $file ]] || die "Guard file already exists: $file"
|
||||
GUARDED_SERVICES+=("$unit"); GUARD_FILES+=("$file")
|
||||
# Record our exact paths BEFORE changing systemd state, including on failures.
|
||||
save_recovery
|
||||
(set -o noclobber; printf '[Unit]\nConditionPathExists=!%s\n' "$SERVICE_GUARD_DIR" > "$file")
|
||||
systemctl daemon-reload
|
||||
paths=$(systemctl show "$unit" --property=DropInPaths --value)
|
||||
# Requiring our drop-in last avoids a later Condition*= resetting the guard.
|
||||
[[ ${paths##* } == "$file" ]] || die "Start guard not loaded last for $unit; review its drop-ins"
|
||||
echo "-- Start guard installed: $unit"
|
||||
}
|
||||
|
||||
assert_services_quiescent() {
|
||||
local i unit state stamp paths
|
||||
for i in "${!GUARDED_SERVICES[@]}"; do
|
||||
unit=${GUARDED_SERVICES[i]}
|
||||
state=$(systemctl show "$unit" --property=ActiveState --value)
|
||||
[[ $state == inactive || $state == failed ]] || die "Consistency lost: writer $unit became $state during the copy window"
|
||||
if [[ -n ${GUARD_FILES[i]} ]]; then
|
||||
[[ -d $SERVICE_GUARD_DIR && -f ${GUARD_FILES[i]} ]] || die "Consistency lost: start guard removed for $unit"
|
||||
[[ $(cat -- "${GUARD_FILES[i]}") == "$(printf '[Unit]\nConditionPathExists=!%s' "$SERVICE_GUARD_DIR")" ]] || die "Consistency lost: start guard changed for $unit"
|
||||
paths=$(systemctl show "$unit" --property=DropInPaths --value)
|
||||
[[ ${paths##* } == "${GUARD_FILES[i]}" ]] || die "Consistency lost: start guard overridden for $unit"
|
||||
else
|
||||
[[ $(systemctl show "$unit" --property=LoadState --value) == masked ]] || die "Consistency lost: existing mask removed for $unit"
|
||||
fi
|
||||
stamp=$(systemctl show "$unit" --property=ActiveEnterTimestampMonotonic --property=InactiveExitTimestampMonotonic)
|
||||
if [[ -n ${GUARD_STAMPS[i]:-} && $stamp != "${GUARD_STAMPS[i]}" ]]; then
|
||||
die "Consistency lost: writer $unit was restarted during the copy window"
|
||||
fi
|
||||
[[ -n $stamp ]] || die "Missing writer state timestamps: $unit"
|
||||
GUARD_STAMPS[i]=$stamp
|
||||
done
|
||||
}
|
||||
|
||||
release_service_guards() {
|
||||
local file
|
||||
# Remove only this run's drop-ins. Keep the condition directory until reload
|
||||
# succeeds, so stale manager state continues to refuse starts on failure.
|
||||
for file in "${GUARD_FILES[@]}"; do
|
||||
[[ -z $file ]] || rm -f -- "$file" || return 1
|
||||
done
|
||||
if (( ${#GUARDED_SERVICES[@]} )); then systemctl daemon-reload || return 1; fi
|
||||
if [[ -n $SERVICE_GUARD_DIR ]]; then rmdir -- "$SERVICE_GUARD_DIR" || return 1; fi
|
||||
SERVICE_GUARD_DIR=''; GUARDED_SERVICES=(); GUARD_FILES=(); GUARD_STAMPS=()
|
||||
}
|
||||
|
||||
stop_service() {
|
||||
local state
|
||||
install_service_guard "$1"
|
||||
state=$(systemctl show "$1" --property=ActiveState --value)
|
||||
case $state in
|
||||
active|activating|reloading)
|
||||
@@ -330,15 +407,19 @@ quiesce() {
|
||||
helper wait-nextcloud "$NC_DIR" "${WRITER_DRAIN_TIMEOUT:=300}"
|
||||
echo '-- Nextcloud writers finished'
|
||||
fi
|
||||
assert_services_quiescent
|
||||
echo '-- All configured writers stopped and protected against activation'
|
||||
}
|
||||
# Survives SIGKILL/power loss, which Bash traps cannot handle. This is an
|
||||
# instruction record for the operator, not an automatically executed recovery.
|
||||
save_recovery() {
|
||||
python3 - "$WORKDIR/recovery-$RUN_ID.json" "$OPERATION" "$NC_CHANGED" "$NC_WAS_ON" "${STAGING_DIR:-}" "${STOPPED_SERVICES[@]}" <<'PYREC'
|
||||
python3 - "$WORKDIR/recovery-$RUN_ID.json" "$OPERATION" "$NC_CHANGED" "$NC_WAS_ON" "${STAGING_DIR:-}" "$SERVICE_GUARD_DIR" "${#GUARD_FILES[@]}" "${GUARD_FILES[@]}" "${STOPPED_SERVICES[@]}" <<'PYREC'
|
||||
import json,os,sys
|
||||
path,operation,changed,was_on,staging,*services=sys.argv[1:]
|
||||
path,operation,changed,was_on,staging,guard_dir,count,*items=sys.argv[1:]
|
||||
guard_files=items[:int(count)]
|
||||
services=items[int(count):]
|
||||
with open(path+'.tmp','w') as stream:
|
||||
json.dump(dict(operation=operation, staging=staging, nextcloud_changed=changed=='true', nextcloud_was_on=was_on=='true', stopped_services=services),stream,indent=2)
|
||||
json.dump(dict(operation=operation, staging=staging, nextcloud_changed=changed=='true', nextcloud_was_on=was_on=='true', stopped_services=services, service_guard_directory=guard_dir, service_guard_files=[p for p in guard_files if p]),stream,indent=2)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(path+'.tmp',path)
|
||||
@@ -346,16 +427,27 @@ PYREC
|
||||
}
|
||||
resume_services() {
|
||||
local failed=0 i
|
||||
# Remove all our guards before starting dependencies in reverse order. Never
|
||||
# start anything if guard cleanup/reload failed: retain manual recovery state.
|
||||
if ! release_service_guards; then
|
||||
echo 'ERROR: Could not release writer start guards; manual recovery required'
|
||||
save_recovery
|
||||
return 1
|
||||
fi
|
||||
if [[ $NC_CHANGED == true ]]; then
|
||||
if occ maintenance:mode --off; then NC_CHANGED=false; else failed=1; fi
|
||||
fi
|
||||
for ((i=${#STOPPED_SERVICES[@]}-1; i>=0; i--)); do
|
||||
[[ -n ${STOPPED_SERVICES[i]} ]] || continue
|
||||
echo "-- Starting service: ${STOPPED_SERVICES[i]} (waiting for systemd)"
|
||||
if systemctl start "${STOPPED_SERVICES[i]}"; then
|
||||
if systemctl start "${STOPPED_SERVICES[i]}" &&
|
||||
[[ $(systemctl show "${STOPPED_SERVICES[i]}" --property=ActiveState --value) == active ]]; then
|
||||
echo "-- Service started: ${STOPPED_SERVICES[i]}"
|
||||
STOPPED_SERVICES[i]=''
|
||||
else failed=1; fi
|
||||
else
|
||||
echo "ERROR: Service did not become active: ${STOPPED_SERVICES[i]}"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
if (( failed == 0 )); then
|
||||
rm -f -- "$WORKDIR/recovery-$RUN_ID.json"
|
||||
|
||||
Reference in new issue
Block a user