johannes/hermes-arch-stack
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sync workstation helpers and preserve baseline deployment

Browse Source
This commit is contained in:
admin committed 2026-09-09 13:31:47 +02:00
1 parent 8641464a6c
commit 3c1080e152
15 files changed
+282 -138
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

libexec/hermes-stack-health
+72 -36
View File
@@ -1,55 +1,91 @@
#!/usr/bin/env bash
set -Euo pipefail
set -Eeuo pipefail
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env"
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/hermes-stack-maintenance"
NOTIFIER="$HOME/.local/libexec/hermes-stack-notify"
mkdir -p "$STATE_DIR"
# shellcheck disable=SC1090
source "$CONFIG_FILE"
# Niemals den absichtlich gestoppten Gateway mitten im Update neu starten.
exec 9>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
log() {
printf '%s\n' "$*" | systemd-cat -t hermes-stack-maintenance
}
notify_user() {
if ! "${NOTIFIER:-$(dirname -- "${BASH_SOURCE[0]}")/hermes-stack-notify}" "$1"; then
log "$1" || true
log "Benachrichtigungshelfer fehlgeschlagen; Healthcheck wird fortgesetzt." || true
fi
return 0
}
LOCK_FILE="${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-health.lock"
exec 9>"$LOCK_FILE"
flock -n 9 || exit 0
state_file="$STATE_DIR/health-state"
problems=()
# Nicht mit Hermes-Update oder kontrolliertem Backend-Neustart kollidieren.
exec 8>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
flock -n 8 || exit 0
wait_for_llama() {
local attempts="${1:-90}"
local i
for ((i=1; i<=attempts; i++)); do
if curl --fail --silent --max-time 5 "$LLAMA_URL/health" |
jq -e '.status == "ok"' >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
failed=0
messages=()
if systemctl --user is-active --quiet "$LLAMA_SERVICE"; then
if ! wait_for_llama 3; then
messages+=("llama-server antwortete nicht gesund; kein automatischer Neustart laufender Inferenz.")
failed=1
fi
elif [[ "${LLAMA_HEALTH_AUTORESTART:-0}" == "1" ]]; then
messages+=("$LLAMA_SERVICE war inaktiv; automatischer Start wurde versucht.")
if ! systemctl --user start "$LLAMA_SERVICE"; then
failed=1
elif ! wait_for_llama 3; then
messages+=("llama-server ist nach dem Start noch nicht gesund; kein weiterer Neustart.")
failed=1
fi
else
messages+=("$LLAMA_SERVICE ist inaktiv; automatischer Start ist deaktiviert.")
failed=1
fi
if ! systemctl --user is-active --quiet "$GATEWAY_SERVICE"; then
problems+=("$GATEWAY_SERVICE war inaktiv; Neustart wurde versucht")
systemctl --user restart "$GATEWAY_SERVICE" || true
messages+=("$GATEWAY_SERVICE war inaktiv; Neustart wurde versucht.")
systemctl --user restart "$GATEWAY_SERVICE" || failed=1
sleep 5
systemctl --user is-active --quiet "$GATEWAY_SERVICE" || failed=1
fi
if ! hermes gateway status >/dev/null 2>&1; then
problems+=("Hermes-Gateway antwortete nicht gesund; Neustart wurde versucht")
systemctl --user restart "$GATEWAY_SERVICE" || true
if [[ -n "${DOCKER_NETWORK:-}" ]] &&
! docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1; then
messages+=("Docker-Netz $DOCKER_NETWORK fehlt.")
failed=1
fi
if [[ "${LLAMA_HEALTH_AUTORESTART:-0}" == 1 ]] &&
! systemctl --user is-active --quiet "$LLAMA_SERVICE"; then
problems+=("$LLAMA_SERVICE war inaktiv; Neustart wurde versucht")
systemctl --user restart "$LLAMA_SERVICE" || true
status_file="$STATE_DIR/health-status"
previous="$(cat "$status_file" 2>/dev/null || true)"
if (( failed )); then
printf 'failed\n' >"$status_file"
notify_user "Hermes-Stack: Gesundheitsprüfung fehlgeschlagen. ${messages[*]} Details: journalctl --user -u $LLAMA_SERVICE -u $GATEWAY_SERVICE"
exit 1
fi
if ! curl --fail --silent --max-time 10 "$LLAMA_URL/health" |
jq -e '.status == "ok"' >/dev/null 2>&1; then
problems+=("llama-server antwortete nicht gesund; kein HTTP-bedingter Neustart laufender Inferenz")
printf 'ok\n' >"$status_file"
if [[ "$previous" == "failed" ]]; then
notify_user "Hermes-Stack: Dienste sind wieder gesund."
elif ((${#messages[@]})); then
notify_user "Hermes-Stack: automatische Wiederherstellung erfolgreich. ${messages[*]}"
fi
if ((${#problems[@]} == 0)); then
current="healthy"
previous="$(cat "$state_file" 2>/dev/null || true)"
if [[ "$previous" == unhealthy ]]; then
"$NOTIFIER" "Hermes-Stack: Dienste sind wieder gesund." || true
fi
else
current="unhealthy"
previous="$(cat "$state_file" 2>/dev/null || true)"
if [[ "$previous" != unhealthy ]]; then
message="$(IFS='; '; printf '%s' "${problems[*]}")"
"$NOTIFIER" "Hermes-Stack: Gesundheitsprüfung fehlgeschlagen. $message. Details: journalctl --user -u $LLAMA_SERVICE -u $GATEWAY_SERVICE" || true
fi
fi
printf '%s\n' "$current" >"$state_file"
[[ "$current" == healthy ]]
libexec/hermes-stack-llama-profile
Regular → Executable
+30 -11
View File
@@ -1,35 +1,54 @@
#!/usr/bin/env bash
set -Eeuo pipefail
umask 077
config_dir="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance"
runner="$(dirname -- "${BASH_SOURCE[0]}")/hermes-stack-llama-run"
case "${1:-list}" in
list)
printf '%-12s %8s %10s\n' Profil Kontext Fit-MiB \
baseline 65536 1536 balanced 49152 1536 vram-plus 49152 1280 compact 32768 1536
baseline 65536 1536 vram-64k 65536 1280 \
balanced 49152 1536 vram-plus 49152 1280 compact 32768 1536
exit 0 ;;
baseline|balanced|vram-plus|compact) selected="$1" ;;
*) printf 'Aufruf: %s [list|baseline|balanced|vram-plus|compact]\n' "$0" >&2; exit 2 ;;
baseline|vram-64k|balanced|vram-plus|compact) selected="$1" ;;
*)
printf 'Unbekanntes Profil. Verfügbare Profile: %s list\n' "$0" >&2
exit 2 ;;
esac
[[ $# -eq 1 ]] || exit 2
# Profilwechsel nur für bewusst migrierte Konfigurationen erlauben.
exec 9>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
flock -n 9 || {
printf 'Maintenance läuft; nichts geändert.\n' >&2
exit 1
}
# shellcheck disable=SC1090
source "$config_dir/llama.env"
[[ -n "${LLAMA_PROFILE:-}" ]] || {
printf 'Zuerst llama.env nach docs/WORKSTATION.md migrieren. Keine Änderung.\n' >&2; exit 1;
printf 'Profilbetrieb ist nicht konfiguriert.\n' >&2
exit 1
}
"$runner" --profile "$selected" --check
exec 9>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
flock -n 9 || { printf 'Maintenance läuft. Später erneut versuchen.\n' >&2; exit 1; }
target="$config_dir/llama-profile.env"
[[ ! -L "$target" ]] || {
printf 'Abbruch: Profilwahl ist ein Symlink.\n' >&2
exit 1
}
if [[ -e "$target" ]]; then
[[ -f "$target" ]] || exit 1
backup="$(mktemp "$config_dir/llama-profile.backup.XXXXXX")"
cp -p -- "$target" "$backup"
printf 'Vorherige Profilwahl: %s\n' "$backup"
printf 'Vorherige Profilwahl gesichert: %s\n' "$backup"
fi
candidate="$(mktemp "$config_dir/llama-profile.env.XXXXXX")"
candidate="$(mktemp "$config_dir/llama-profile.env.pending.XXXXXX")"
trap 'rm -f -- "$candidate"' EXIT
printf 'LLAMA_PROFILE=%q\n' "$selected" >"$candidate"
mv -T -- "$candidate" "$target"
printf 'Profil %s gespeichert. Laufende Jobs bleiben unverändert.\n' "$selected"
printf 'Erst nach Jobende: hermes-stack-llama-restart (siehe docs/WORKSTATION.md).\n'
printf 'Profil %s für den nächsten Dienststart gespeichert. Kein Neustart.\n' "$selected"
libexec/hermes-stack-llama-report
Regular → Executable
View File
File mode changed.
libexec/hermes-stack-llama-restart
Regular → Executable
+15 -4
View File
@@ -1,14 +1,25 @@
#!/usr/bin/env bash
set -Eeuo pipefail
if [[ "${1:-}" != --jobs-finished || $# -ne 1 ]]; then
printf 'Stoppt laufende Inferenz! Erst nach Jobende ausführen:\n %s --jobs-finished\n' "$0" >&2
printf 'Dieser Neustart unterbricht laufende Backend-Jobs.\n' >&2
printf 'Erst nach Jobende ausführen: %s --jobs-finished\n' "$0" >&2
exit 2
fi
config_dir="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance"
# shellcheck disable=SC1090
source "$config_dir/config.env"
exec 9>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
flock -n 9 || { printf 'Maintenance läuft; kein Neustart.\n' >&2; exit 1; }
exec 8>"${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
if ! flock -n 8; then
printf 'Maintenance läuft gerade. Kein Neustart ausgeführt.\n' >&2
exit 1
fi
"$(dirname -- "${BASH_SOURCE[0]}")/hermes-stack-llama-run" --check
printf 'Starte %s neu …\n' "$LLAMA_SERVICE"
systemctl --user restart "$LLAMA_SERVICE"
printf 'Neustart ausgelöst. Modell kann noch laden; Logs und Inferenz prüfen.\n'
printf 'Neustart ausgelöst. Das Modell kann noch laden.\n'
libexec/hermes-stack-llama-run
+40 -10
View File
@@ -13,11 +13,19 @@ while (($#)); do
case "$1" in
--check) mode=check ;;
--print) mode=print ;;
--profile) [[ $# -ge 2 ]] || exit 2; profile_override="$2"; shift ;;
*) printf 'Aufruf: %s [--check|--print] [--profile NAME]\n' "$0" >&2; exit 2 ;;
--profile)
[[ $# -ge 2 ]] || exit 2
profile_override="$2"
shift
;;
*)
printf 'Aufruf: %s [--check|--print] [--profile NAME]\n' "$0" >&2
exit 2
;;
esac
shift
done
if [[ -n "${LLAMA_PROFILE:-}" && -r "$config_dir/llama-profile.env" ]]; then
# shellcheck disable=SC1090
source "$config_dir/llama-profile.env"
@@ -30,6 +38,7 @@ if [[ -z "${LLAMA_SERVER_BIN:-}" ]]; then
LLAMA_SERVER_BIN="$(command -v llama-server || true)"
fi
fi
[[ -x "$LLAMA_SERVER_BIN" ]] || {
printf 'llama-server nicht ausführbar: %s\n' "$LLAMA_SERVER_BIN" >&2
exit 1
@@ -40,7 +49,10 @@ if [[ -n "${LLAMA_MODEL:-}" && -n "${LLAMA_HF_REPO:-}" ]]; then
printf 'Nur LLAMA_MODEL ODER LLAMA_HF_REPO setzen.\n' >&2
exit 1
elif [[ -n "${LLAMA_MODEL:-}" ]]; then
[[ -r "$LLAMA_MODEL" ]] || { printf 'Modell nicht lesbar.\n' >&2; exit 1; }
[[ -r "$LLAMA_MODEL" ]] || {
printf 'Modell nicht lesbar.\n' >&2
exit 1
}
model_args=(--model "$LLAMA_MODEL")
elif [[ -n "${LLAMA_HF_REPO:-}" ]]; then
model_args=(--hf-repo "$LLAMA_HF_REPO")
@@ -48,8 +60,12 @@ else
printf 'LLAMA_MODEL oder LLAMA_HF_REPO fehlt.\n' >&2
exit 1
fi
if [[ -n "${LLAMA_MMPROJ:-}" ]]; then
[[ -r "$LLAMA_MMPROJ" ]] || { printf 'Projektor nicht lesbar.\n' >&2; exit 1; }
[[ -r "$LLAMA_MMPROJ" ]] || {
printf 'Projektor nicht lesbar.\n' >&2
exit 1
}
model_args+=(--mmproj "$LLAMA_MMPROJ")
fi
[[ -z "${LLAMA_ALIAS:-}" ]] || model_args+=(--alias "$LLAMA_ALIAS")
@@ -58,11 +74,16 @@ profile_args=()
if [[ -n "${LLAMA_PROFILE:-}" ]]; then
case "$LLAMA_PROFILE" in
baseline) ctx=65536; margin=1536 ;;
vram-64k) ctx=65536; margin=1280 ;;
balanced) ctx=49152; margin=1536 ;;
vram-plus) ctx=49152; margin=1280 ;;
compact) ctx=32768; margin=1536 ;;
*) printf 'Unbekanntes Profil: %s\n' "$LLAMA_PROFILE" >&2; exit 2 ;;
*)
printf 'Unbekanntes Profil: %s\n' "$LLAMA_PROFILE" >&2
exit 2
;;
esac
profile_args=(
--ctx-size "$ctx" --parallel 1 --jinja
--reasoning auto --reasoning-budget 512 --reasoning-preserve
@@ -72,25 +93,33 @@ if [[ -n "${LLAMA_PROFILE:-}" ]]; then
--sleep-idle-seconds 300 --image-min-tokens 1024
--cors-origins localhost --no-ui
)
# Alte vollständige LLAMA_ARGS nicht still mit einem Profil vermischen.
if ((${#LLAMA_ARGS[@]})); then
printf 'Bei Profilbetrieb LLAMA_ARGS leeren; eigene vollständige Args nur ohne LLAMA_PROFILE.\n' >&2
exit 2
fi
fi
args=("${model_args[@]}" --host "${LLAMA_HOST:-127.0.0.1}" --port "${LLAMA_PORT:-8080}"
"${profile_args[@]}" "${LLAMA_ARGS[@]}")
args=(
"${model_args[@]}"
--host "${LLAMA_HOST:-127.0.0.1}"
--port "${LLAMA_PORT:-8080}"
"${profile_args[@]}"
"${LLAMA_ARGS[@]}"
)
if [[ "$mode" == print ]]; then
printf '%q ' "$LLAMA_SERVER_BIN" "${args[@]}"
printf '\n'
exit 0
fi
# Kein Download/Modellstart: nur prüfen, ob die installierte Version die Namen kennt.
# Nur Optionsnamen prüfen: kein Modellstart oder Download.
help_text="$(timeout 30 "$LLAMA_SERVER_BIN" --help 2>&1)" || {
printf 'llama-server --help fehlgeschlagen.\n' >&2; exit 1;
printf 'llama-server --help fehlgeschlagen.\n' >&2
exit 1
}
for arg in "${args[@]}"; do
[[ "$arg" == --* ]] || continue
option="${arg%%=*}"
@@ -99,6 +128,7 @@ for arg in "${args[@]}"; do
exit 1
fi
done
if [[ "$mode" == check ]]; then
printf 'Konfiguration und Optionsnamen OK; GPU/Modell/Optionswerte noch nicht laufzeitgetestet.\n'
exit 0
libexec/hermes-stack-notify
+72 -71
View File
@@ -1,83 +1,84 @@
#!/usr/bin/env bash
set -Euo pipefail
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env"
log() {
printf '%s\n' "$*" | systemd-cat -t hermes-stack-maintenance 2>/dev/null || true
}
text="${*:-}"
if [[ -z "$text" ]]; then
printf 'Verwendung: %s TEXT\n' "${0##*/}" >&2
exit 2
fi
log "$text"
[[ -n "${*:-}" ]] || exit 2
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env"
if [[ ! -r "$CONFIG_FILE" ]]; then
log "Benachrichtigung übersprungen: $CONFIG_FILE ist nicht lesbar."
log "$*"
log "Benachrichtigung übersprungen: Maintenance-Konfiguration nicht lesbar."
exit 0
fi
set +u
# shellcheck disable=SC1090
source "$CONFIG_FILE" || {
set -u
log "Benachrichtigung übersprungen: Maintenance-Konfiguration ist ungültig."
exit 0
source "$CONFIG_FILE" || { log "$*"; exit 0; }
notify_user() {
local text="$1"
local telegram_notify="${TELEGRAM_NOTIFY:-0}"
local signal_notify="${SIGNAL_NOTIFY:-0}"
local signal_payload
log "$text"
if [[ -r "${HERMES_ENV:-}" ]] &&
[[ "$telegram_notify" == "1" || "$signal_notify" == "1" ]]; then
set +u
set -a
# shellcheck disable=SC1090
source "$HERMES_ENV"
set +a
set -u
fi
if [[ "$telegram_notify" == "1" &&
-n "${TELEGRAM_BOT_TOKEN:-}" &&
-n "${TELEGRAM_HOME_CHANNEL:-}" ]]; then
curl --fail --silent --show-error --max-time 15 \
--request POST \
"https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
--data-urlencode "chat_id=${TELEGRAM_HOME_CHANNEL}" \
--data-urlencode "text=${text}" \
>/dev/null || true
fi
if [[ "$signal_notify" == "1" &&
-n "${SIGNAL_HTTP_URL:-}" &&
-n "${SIGNAL_ACCOUNT:-}" &&
-n "${SIGNAL_HOME_CHANNEL:-}" ]]; then
if [[ "$SIGNAL_HOME_CHANNEL" == group:* ]]; then
if ! signal_payload="$(
jq -nc \
--arg account "$SIGNAL_ACCOUNT" \
--arg message "$text" \
--arg group_id "${SIGNAL_HOME_CHANNEL#group:}" \
'{jsonrpc:"2.0",method:"send",params:{account:$account,message:$message,groupId:$group_id},id:"hermes-maintenance"}'
)"; then
log "Signal-Benachrichtigung konnte nicht vorbereitet werden."
return 0
fi
else
if ! signal_payload="$(
jq -nc \
--arg account "$SIGNAL_ACCOUNT" \
--arg message "$text" \
--arg recipient "$SIGNAL_HOME_CHANNEL" \
'{jsonrpc:"2.0",method:"send",params:{account:$account,message:$message,recipient:[$recipient]},id:"hermes-maintenance"}'
)"; then
log "Signal-Benachrichtigung konnte nicht vorbereitet werden."
return 0
fi
fi
if ! curl --fail --silent --show-error --max-time 30 \
--header 'Content-Type: application/json' \
--data-binary "$signal_payload" \
"${SIGNAL_HTTP_URL%/}/api/v1/rpc" |
jq -e '((.error // null) == null) and (.result != null)' \
>/dev/null; then
log "Signal-Benachrichtigung konnte nicht zugestellt werden."
fi
fi
}
telegram_notify="${TELEGRAM_NOTIFY:-0}"
signal_notify="${SIGNAL_NOTIFY:-0}"
if [[ -r "${HERMES_ENV:-}" ]] &&
[[ "$telegram_notify" == 1 || "$signal_notify" == 1 ]]; then
set -a
# shellcheck disable=SC1090
source "$HERMES_ENV" || true
set +a
fi
set -u
if [[ "$telegram_notify" == 1 &&
-n "${TELEGRAM_BOT_TOKEN:-}" &&
-n "${TELEGRAM_HOME_CHANNEL:-}" ]]; then
if ! curl --fail --silent --show-error --max-time 15 \
--request POST \
"https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
--data-urlencode "chat_id=${TELEGRAM_HOME_CHANNEL}" \
--data-urlencode "text=${text}" >/dev/null; then
log "Telegram-Benachrichtigung konnte nicht zugestellt werden."
fi
fi
if [[ "$signal_notify" == 1 &&
-n "${SIGNAL_HTTP_URL:-}" &&
-n "${SIGNAL_ACCOUNT:-}" &&
-n "${SIGNAL_HOME_CHANNEL:-}" ]]; then
if [[ "$SIGNAL_HOME_CHANNEL" == group:* ]]; then
signal_payload="$(
jq -nc --arg account "$SIGNAL_ACCOUNT" --arg message "$text" \
--arg group_id "${SIGNAL_HOME_CHANNEL#group:}" \
'{jsonrpc:"2.0",method:"send",params:{account:$account,message:$message,groupId:$group_id},id:"hermes-maintenance"}'
)" || signal_payload=""
else
signal_payload="$(
jq -nc --arg account "$SIGNAL_ACCOUNT" --arg message "$text" \
--arg recipient "$SIGNAL_HOME_CHANNEL" \
'{jsonrpc:"2.0",method:"send",params:{account:$account,message:$message,recipient:[$recipient]},id:"hermes-maintenance"}'
)" || signal_payload=""
fi
if [[ -z "$signal_payload" ]] ||
! curl --fail --silent --show-error --max-time 30 \
--header 'Content-Type: application/json' \
--data-binary "$signal_payload" \
"${SIGNAL_HTTP_URL%/}/api/v1/rpc" |
jq -e '((.error // null) == null) and (.result != null)' >/dev/null; then
log "Signal-Benachrichtigung konnte nicht zugestellt werden."
fi
fi
notify_user "$*" || true
exit 0