92 lines
2.7 KiB
Bash
Executable File
92 lines
2.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env"
|
|
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/hermes-stack-maintenance"
|
|
mkdir -p "$STATE_DIR"
|
|
# shellcheck disable=SC1090
|
|
source "$CONFIG_FILE"
|
|
|
|
log() {
|
|
printf '%s\n' "$*" | systemd-cat -t hermes-stack-maintenance
|
|
}
|
|
|
|
notify_user() {
|
|
if ! "${NOTIFIER:-$(dirname -- "${BASH_SOURCE[0]}")/hermes-stack-notify}" "$1"; then
|
|
log "$1" || true
|
|
log "Benachrichtigungshelfer fehlgeschlagen; Healthcheck wird fortgesetzt." || true
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
LOCK_FILE="${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-health.lock"
|
|
exec 9>"$LOCK_FILE"
|
|
flock -n 9 || exit 0
|
|
|
|
# Nicht mit Hermes-Update oder kontrolliertem Backend-Neustart kollidieren.
|
|
exec 8>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
|
|
flock -n 8 || exit 0
|
|
|
|
wait_for_llama() {
|
|
local attempts="${1:-90}"
|
|
local i
|
|
for ((i=1; i<=attempts; i++)); do
|
|
if curl --fail --silent --max-time 5 "$LLAMA_URL/health" |
|
|
jq -e '.status == "ok"' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
failed=0
|
|
messages=()
|
|
|
|
if systemctl --user is-active --quiet "$LLAMA_SERVICE"; then
|
|
if ! wait_for_llama 3; then
|
|
messages+=("llama-server antwortete nicht gesund; kein automatischer Neustart laufender Inferenz.")
|
|
failed=1
|
|
fi
|
|
elif [[ "${LLAMA_HEALTH_AUTORESTART:-0}" == "1" ]]; then
|
|
messages+=("$LLAMA_SERVICE war inaktiv; automatischer Start wurde versucht.")
|
|
if ! systemctl --user start "$LLAMA_SERVICE"; then
|
|
failed=1
|
|
elif ! wait_for_llama 3; then
|
|
messages+=("llama-server ist nach dem Start noch nicht gesund; kein weiterer Neustart.")
|
|
failed=1
|
|
fi
|
|
else
|
|
messages+=("$LLAMA_SERVICE ist inaktiv; automatischer Start ist deaktiviert.")
|
|
failed=1
|
|
fi
|
|
|
|
if ! systemctl --user is-active --quiet "$GATEWAY_SERVICE"; then
|
|
messages+=("$GATEWAY_SERVICE war inaktiv; Neustart wurde versucht.")
|
|
systemctl --user restart "$GATEWAY_SERVICE" || failed=1
|
|
sleep 5
|
|
systemctl --user is-active --quiet "$GATEWAY_SERVICE" || failed=1
|
|
fi
|
|
|
|
if [[ -n "${DOCKER_NETWORK:-}" ]] &&
|
|
! docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1; then
|
|
messages+=("Docker-Netz $DOCKER_NETWORK fehlt.")
|
|
failed=1
|
|
fi
|
|
|
|
status_file="$STATE_DIR/health-status"
|
|
previous="$(cat "$status_file" 2>/dev/null || true)"
|
|
|
|
if (( failed )); then
|
|
printf 'failed\n' >"$status_file"
|
|
notify_user "Hermes-Stack: Gesundheitsprüfung fehlgeschlagen. ${messages[*]} Details: journalctl --user -u $LLAMA_SERVICE -u $GATEWAY_SERVICE"
|
|
exit 1
|
|
fi
|
|
|
|
printf 'ok\n' >"$status_file"
|
|
if [[ "$previous" == "failed" ]]; then
|
|
notify_user "Hermes-Stack: Dienste sind wieder gesund."
|
|
elif ((${#messages[@]})); then
|
|
notify_user "Hermes-Stack: automatische Wiederherstellung erfolgreich. ${messages[*]}"
|
|
fi
|