#!/usr/bin/env bash set -Eeuo pipefail CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env" STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/hermes-stack-maintenance" NOTIFIER="$HOME/.local/libexec/hermes-stack-notify" mkdir -p "$STATE_DIR" # shellcheck disable=SC1090 source "$CONFIG_FILE" log() { printf '%s\n' "$*" | systemd-cat -t hermes-stack-maintenance } notify_user() { local text="$1" if [[ -x "$NOTIFIER" ]]; then "$NOTIFIER" "$text" || true else log "$text" fi } fetch_origin_main_with_retry() { local -a retry_delays=(0 5 20) local -a git_options=() local attempt delay for ((attempt = 1; attempt <= ${#retry_delays[@]}; attempt++)); do delay="${retry_delays[attempt - 1]}" if ((delay > 0)); then log "Hermes-Git-Fetch: neuer Versuch $attempt/${#retry_delays[@]} in ${delay}s." sleep "$delay" fi git_options=() if ((attempt == ${#retry_delays[@]})); then git_options=(-c http.version=HTTP/1.1) log "Hermes-Git-Fetch: letzter Versuch mit HTTP/1.1." fi if GIT_TERMINAL_PROMPT=0 timeout --foreground 90s \ git "${git_options[@]}" -C "$HERMES_REPO" fetch --no-tags \ "$HERMES_REMOTE" \ "+refs/heads/$HERMES_BRANCH:refs/remotes/$HERMES_REMOTE/$HERMES_BRANCH"; then ((attempt == 1)) || log "Hermes-Git-Fetch war im Versuch $attempt/${#retry_delays[@]} erfolgreich." return 0 fi log "Hermes-Git-Fetch fehlgeschlagen (Versuch $attempt/${#retry_delays[@]})." done return 1 } run_hermes_update_attempt() { local attempt="$1" local -a pipe_status printf '\n===== Hermes update: Versuch %s/2 (%s) =====\n' \ "$attempt" "$(date --iso-8601=seconds)" >>"$update_log" set +e hermes update 2>&1 | tee -a "$update_log" pipe_status=("${PIPESTATUS[@]}") set -e ((pipe_status[1] == 0)) || return "${pipe_status[1]}" return "${pipe_status[0]}" } is_retryable_update_fetch_failure() { local current_commit dirty_after current_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)" || return 1 [[ "$current_commit" == "$old_commit" ]] || return 1 dirty_after="$( git -C "$HERMES_REPO" status --porcelain --untracked-files=all | grep -vxF '?? .install_method' || true )" [[ -z "$dirty_after" ]] || return 1 grep -Fq 'Fetching updates' "$update_log" || return 1 grep -Fq 'Pulling updates' "$update_log" && return 1 grep -Eqi \ 'HTTP (429|500|502|503|504)|returned error: (429|500|502|503|504)|rate.?limit|could not read Username|Authentication failed|Could not resolve host|Failed to connect|Connection (timed out|reset)|TLS|early EOF|Failed to fetch updates|Network error' \ "$update_log" } for command in git tar zstd jq curl systemctl timeout flock hermes; do command -v "$command" >/dev/null 2>&1 || { notify_user "Hermes-Update abgebrochen: benötigtes Kommando fehlt: $command" exit 1 } done LOCK_FILE="${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock" exec 9>"$LOCK_FILE" flock -n 9 || exit 0 if [[ ! -d "$HERMES_REPO/.git" ]]; then notify_user "Hermes-Update abgebrochen: Git-Installation nicht gefunden." exit 1 fi case "$HERMES_REPO" in "$HOME"/*/hermes-agent) ;; *) notify_user "Hermes-Update abgebrochen: unsicherer Repository-Pfad: $HERMES_REPO" exit 1 ;; esac dirty="$( git -C "$HERMES_REPO" status --porcelain --untracked-files=all | grep -vxF '?? .install_method' || true )" if [[ -n "$dirty" ]]; then notify_user "Hermes-Update abgebrochen: lokale Änderungen im Hermes-Repository: $dirty" exit 1 fi old_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)" if ! fetch_origin_main_with_retry; then notify_user "Hermes-Updateprüfung fehlgeschlagen: $HERMES_REMOTE/$HERMES_BRANCH ist nicht erreichbar. Gateway und Installation bleiben unverändert." exit 1 fi remote_commit="$(git -C "$HERMES_REPO" rev-parse "refs/remotes/$HERMES_REMOTE/$HERMES_BRANCH")" if [[ ! "$remote_commit" =~ ^[0-9a-f]{40}$ ]]; then notify_user "Hermes-Updateprüfung fehlgeschlagen: Remote lieferte keinen gültigen Commit." exit 1 fi if [[ "$remote_commit" == "$old_commit" ]]; then notify_user "Hermes ist bereits aktuell: $old_commit" exit 0 fi backup_dir="$STATE_DIR/backups" mkdir -p "$backup_dir" timestamp="$(date +%Y%m%d-%H%M%S)" backup="$backup_dir/hermes-$timestamp.tar.zst" update_log="$STATE_DIR/hermes-update-$timestamp.log" paths=("${HERMES_REPO#"$HOME/"}") for path in \ "$HOME/.hermes/config.yaml" "$HOME/.hermes/.env" "$HOME/.hermes/auth.json" \ "${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user/$GATEWAY_SERVICE"; do [[ -e "$path" && "$path" == "$HOME/"* ]] && paths+=("${path#"$HOME/"}") done tar --zstd -cf "$backup" -C "$HOME" "${paths[@]}" rollback() { trap - ERR set +e systemctl --user stop "$GATEWAY_SERVICE" if [[ "$HERMES_REPO" == "$HOME/"*/hermes-agent ]]; then rm -rf -- "$HERMES_REPO" tar --zstd -xf "$backup" -C "$HOME" fi systemctl --user daemon-reload systemctl --user restart "$GATEWAY_SERVICE" notify_user "Hermes-Update zurückgerollt auf $old_commit. Backup: $backup" exit 1 } trap rollback ERR systemctl --user stop "$GATEWAY_SERVICE" if run_hermes_update_attempt 1; then update_exit=0 else update_exit=$? log "Hermes-Updater meldete Exitcode $update_exit." if is_retryable_update_fetch_failure; then log "Transienter Fetch-Fehler erkannt; Hermes-Update wird einmal wiederholt." sleep 20 if run_hermes_update_attempt 2; then update_exit=0 else update_exit=$? log "Hermes-Updater-Retry meldete Exitcode $update_exit." fi fi fi if ! hermes config check >/dev/null 2>&1; then hermes config migrate hermes config check fi systemctl --user daemon-reload systemctl --user reset-failed "$GATEWAY_SERVICE" || true systemctl --user restart "$GATEWAY_SERVICE" gateway_ready=0 for ((attempt = 1; attempt <= ${GATEWAY_HEALTH_ATTEMPTS:-12}; attempt++)); do if systemctl --user is-active --quiet "$GATEWAY_SERVICE" && hermes gateway status >/dev/null 2>&1; then gateway_ready=1 break fi sleep "${GATEWAY_HEALTH_DELAY:-5}" done if ((gateway_ready == 0)); then notify_user "Hermes-Gateway wurde nach dem Update nicht gesund. Rollback wird ausgeführt." false fi if ! curl --fail --silent --max-time 10 "$LLAMA_URL/health" | jq -e '.status == "ok"' >/dev/null; then notify_user "Hermes und Gateway sind aktualisiert, aber der lokale Modell-Endpunkt ist nicht gesund: $LLAMA_URL. Kein Hermes-Rollback, da llama-server unabhängig ist." fi trap - ERR mapfile -t old_backups < <( find "$backup_dir" -maxdepth 1 -type f -name 'hermes-*.tar.zst' \ -printf '%T@ %p\n' | sort -nr | tail -n "+$((${BACKUP_KEEP:-3} + 1))" | cut -d' ' -f2- ) ((${#old_backups[@]} == 0)) || rm -f -- "${old_backups[@]}" new_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)" if [[ "$new_commit" == "$old_commit" ]]; then notify_user "Hermes-Update wurde nicht installiert (Updater-Exitcode $update_exit); der bisherige Commit $new_commit und der frisch gestartete Gateway sind gesund." exit 1 elif ((update_exit != 0)); then notify_user "Hermes erfolgreich aktualisiert: $old_commit -> $new_commit. Eine Updater-Restartwarnung wurde durch den frischen Gateway-Neustart behoben." else notify_user "Hermes erfolgreich aktualisiert: $old_commit -> $new_commit" fi