244 lines
7.7 KiB
Bash
Executable File
244 lines
7.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env"
|
|
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/hermes-stack-maintenance"
|
|
NOTIFIER="$HOME/.local/libexec/hermes-stack-notify"
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
# shellcheck disable=SC1090
|
|
source "$CONFIG_FILE"
|
|
|
|
log() {
|
|
printf '%s\n' "$*" | systemd-cat -t hermes-stack-maintenance
|
|
}
|
|
|
|
notify_user() {
|
|
local text="$1"
|
|
# Keep the outcome in this service's journal as well as the notifier's journal.
|
|
printf '%s\n' "$text"
|
|
if [[ -x "$NOTIFIER" ]]; then
|
|
"$NOTIFIER" "$text" || true
|
|
else
|
|
log "$text"
|
|
fi
|
|
}
|
|
|
|
fetch_origin_main_with_retry() {
|
|
local -a retry_delays=(0 5 20)
|
|
local -a git_options=()
|
|
local attempt delay
|
|
|
|
for ((attempt = 1; attempt <= ${#retry_delays[@]}; attempt++)); do
|
|
delay="${retry_delays[attempt - 1]}"
|
|
if ((delay > 0)); then
|
|
log "Hermes-Git-Fetch: neuer Versuch $attempt/${#retry_delays[@]} in ${delay}s."
|
|
sleep "$delay"
|
|
fi
|
|
|
|
git_options=()
|
|
if ((attempt == ${#retry_delays[@]})); then
|
|
git_options=(-c http.version=HTTP/1.1)
|
|
log "Hermes-Git-Fetch: letzter Versuch mit HTTP/1.1."
|
|
fi
|
|
|
|
if GIT_TERMINAL_PROMPT=0 timeout --foreground 90s \
|
|
git "${git_options[@]}" -C "$HERMES_REPO" fetch --no-tags \
|
|
"$HERMES_REMOTE" \
|
|
"+refs/heads/$HERMES_BRANCH:refs/remotes/$HERMES_REMOTE/$HERMES_BRANCH"; then
|
|
((attempt == 1)) ||
|
|
log "Hermes-Git-Fetch war im Versuch $attempt/${#retry_delays[@]} erfolgreich."
|
|
return 0
|
|
fi
|
|
log "Hermes-Git-Fetch fehlgeschlagen (Versuch $attempt/${#retry_delays[@]})."
|
|
done
|
|
return 1
|
|
}
|
|
|
|
run_hermes_update_attempt() {
|
|
local attempt="$1"
|
|
local -a pipe_status
|
|
|
|
printf '\n===== Hermes update: Versuch %s/2 (%s) =====\n' \
|
|
"$attempt" "$(date --iso-8601=seconds)" >>"$update_log"
|
|
|
|
set +e
|
|
hermes update 2>&1 | tee -a "$update_log"
|
|
pipe_status=("${PIPESTATUS[@]}")
|
|
set -e
|
|
|
|
((pipe_status[1] == 0)) || return "${pipe_status[1]}"
|
|
return "${pipe_status[0]}"
|
|
}
|
|
|
|
is_retryable_update_fetch_failure() {
|
|
local current_commit dirty_after
|
|
current_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)" || return 1
|
|
[[ "$current_commit" == "$old_commit" ]] || return 1
|
|
|
|
dirty_after="$(
|
|
git -C "$HERMES_REPO" status --porcelain --untracked-files=all |
|
|
grep -vxF '?? .install_method' || true
|
|
)"
|
|
[[ -z "$dirty_after" ]] || return 1
|
|
grep -Fq 'Fetching updates' "$update_log" || return 1
|
|
grep -Fq 'Pulling updates' "$update_log" && return 1
|
|
grep -Eqi \
|
|
'HTTP (429|500|502|503|504)|returned error: (429|500|502|503|504)|rate.?limit|could not read Username|Authentication failed|Could not resolve host|Failed to connect|Connection (timed out|reset)|TLS|early EOF|Failed to fetch updates|Network error' \
|
|
"$update_log"
|
|
}
|
|
|
|
for command in git tar zstd jq curl systemctl timeout flock hermes; do
|
|
command -v "$command" >/dev/null 2>&1 || {
|
|
notify_user "Hermes-Update abgebrochen: benötigtes Kommando fehlt: $command"
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
LOCK_FILE="${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
|
|
exec 9>"$LOCK_FILE"
|
|
flock -n 9 || exit 0
|
|
|
|
if [[ ! -d "$HERMES_REPO/.git" ]]; then
|
|
notify_user "Hermes-Update abgebrochen: Git-Installation nicht gefunden."
|
|
exit 1
|
|
fi
|
|
|
|
case "$HERMES_REPO" in
|
|
"$HOME"/*/hermes-agent) ;;
|
|
*)
|
|
notify_user "Hermes-Update abgebrochen: unsicherer Repository-Pfad: $HERMES_REPO"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
dirty="$(
|
|
git -C "$HERMES_REPO" status --porcelain --untracked-files=all |
|
|
grep -vxF '?? .install_method' || true
|
|
)"
|
|
if [[ -n "$dirty" ]]; then
|
|
notify_user "Hermes-Update abgebrochen: lokale Änderungen im Hermes-Repository:
|
|
$dirty"
|
|
exit 1
|
|
fi
|
|
|
|
old_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)"
|
|
|
|
if ! fetch_origin_main_with_retry; then
|
|
notify_user "Hermes-Updateprüfung fehlgeschlagen: $HERMES_REMOTE/$HERMES_BRANCH ist nicht erreichbar. Gateway und Installation bleiben unverändert."
|
|
exit 1
|
|
fi
|
|
|
|
remote_commit="$(git -C "$HERMES_REPO" rev-parse "refs/remotes/$HERMES_REMOTE/$HERMES_BRANCH")"
|
|
if [[ ! "$remote_commit" =~ ^[0-9a-f]{40}$ ]]; then
|
|
notify_user "Hermes-Updateprüfung fehlgeschlagen: Remote lieferte keinen gültigen Commit."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$remote_commit" == "$old_commit" ]]; then
|
|
notify_user "Hermes ist bereits aktuell: $old_commit"
|
|
exit 0
|
|
fi
|
|
|
|
backup_dir="$STATE_DIR/backups"
|
|
mkdir -p "$backup_dir"
|
|
timestamp="$(date +%Y%m%d-%H%M%S)"
|
|
backup="$backup_dir/hermes-$timestamp.tar.zst"
|
|
update_log="$STATE_DIR/hermes-update-$timestamp.log"
|
|
|
|
paths=("${HERMES_REPO#"$HOME/"}")
|
|
for path in \
|
|
"$HOME/.hermes/config.yaml" "$HOME/.hermes/.env" "$HOME/.hermes/auth.json" \
|
|
"${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user/$GATEWAY_SERVICE"; do
|
|
[[ -e "$path" && "$path" == "$HOME/"* ]] && paths+=("${path#"$HOME/"}")
|
|
done
|
|
|
|
tar --zstd -cf "$backup" -C "$HOME" "${paths[@]}"
|
|
|
|
rollback() {
|
|
trap - ERR
|
|
set +e
|
|
systemctl --user stop "$GATEWAY_SERVICE"
|
|
if [[ "$HERMES_REPO" == "$HOME/"*/hermes-agent ]]; then
|
|
rm -rf -- "$HERMES_REPO"
|
|
tar --zstd -xf "$backup" -C "$HOME"
|
|
fi
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart "$GATEWAY_SERVICE"
|
|
notify_user "Hermes-Update zurückgerollt auf $old_commit. Backup: $backup"
|
|
exit 1
|
|
}
|
|
trap rollback ERR
|
|
|
|
systemctl --user stop "$GATEWAY_SERVICE"
|
|
|
|
if run_hermes_update_attempt 1; then
|
|
update_exit=0
|
|
else
|
|
update_exit=$?
|
|
log "Hermes-Updater meldete Exitcode $update_exit."
|
|
if is_retryable_update_fetch_failure; then
|
|
log "Transienter Fetch-Fehler erkannt; Hermes-Update wird einmal wiederholt."
|
|
sleep 20
|
|
if run_hermes_update_attempt 2; then
|
|
update_exit=0
|
|
else
|
|
update_exit=$?
|
|
log "Hermes-Updater-Retry meldete Exitcode $update_exit."
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if ! hermes config check >/dev/null 2>&1; then
|
|
hermes config migrate
|
|
hermes config check
|
|
fi
|
|
|
|
systemctl --user daemon-reload
|
|
systemctl --user reset-failed "$GATEWAY_SERVICE" || true
|
|
systemctl --user restart "$GATEWAY_SERVICE"
|
|
|
|
gateway_ready=0
|
|
for ((attempt = 1; attempt <= ${GATEWAY_HEALTH_ATTEMPTS:-12}; attempt++)); do
|
|
if systemctl --user is-active --quiet "$GATEWAY_SERVICE" &&
|
|
hermes gateway status >/dev/null 2>&1; then
|
|
gateway_ready=1
|
|
break
|
|
fi
|
|
sleep "${GATEWAY_HEALTH_DELAY:-5}"
|
|
done
|
|
|
|
if ((gateway_ready == 0)); then
|
|
notify_user "Hermes-Gateway wurde nach dem Update nicht gesund. Rollback wird ausgeführt."
|
|
false
|
|
fi
|
|
|
|
if ! curl --fail --silent --max-time 10 "$LLAMA_URL/health" |
|
|
jq -e '.status == "ok"' >/dev/null; then
|
|
notify_user "Hermes und Gateway sind aktualisiert, aber der lokale Modell-Endpunkt ist nicht gesund: $LLAMA_URL. Kein Hermes-Rollback, da llama-server unabhängig ist."
|
|
fi
|
|
|
|
trap - ERR
|
|
|
|
mapfile -t old_backups < <(
|
|
find "$backup_dir" -maxdepth 1 -type f -name 'hermes-*.tar.zst' \
|
|
-printf '%T@ %p\n' | sort -nr | tail -n "+$((${BACKUP_KEEP:-3} + 1))" |
|
|
cut -d' ' -f2-
|
|
)
|
|
((${#old_backups[@]} == 0)) || rm -f -- "${old_backups[@]}"
|
|
|
|
new_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)"
|
|
# The updater selects the configured channel (e.g. stable), whereas the fetch
|
|
# above only checks remote connectivity. Stable may legitimately stay on the
|
|
# same commit even when origin/main has advanced.
|
|
if [[ "$new_commit" == "$old_commit" ]] && ((update_exit == 0)); then
|
|
notify_user "Hermes ist im konfigurierten Update-Kanal bereits aktuell: $new_commit. Konfiguration und frisch gestarteter Gateway sind gesund."
|
|
elif [[ "$new_commit" == "$old_commit" ]]; then
|
|
notify_user "Hermes-Update wurde nicht installiert (Updater-Exitcode $update_exit); der bisherige Commit $new_commit und der frisch gestartete Gateway sind gesund."
|
|
exit 1
|
|
elif ((update_exit != 0)); then
|
|
notify_user "Hermes erfolgreich aktualisiert: $old_commit -> $new_commit. Eine Updater-Restartwarnung wurde durch den frischen Gateway-Neustart behoben."
|
|
else
|
|
notify_user "Hermes erfolgreich aktualisiert: $old_commit -> $new_commit"
|
|
fi
|