Initial commit
This commit is contained in:
admin committed
commit
8641464a6c
44 files changed
+2192
No files matched your search
Executable
+236
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance/config.env"
|
||||
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/hermes-stack-maintenance"
|
||||
NOTIFIER="$HOME/.local/libexec/hermes-stack-notify"
|
||||
mkdir -p "$STATE_DIR"
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
|
||||
log() {
|
||||
printf '%s\n' "$*" | systemd-cat -t hermes-stack-maintenance
|
||||
}
|
||||
|
||||
notify_user() {
|
||||
local text="$1"
|
||||
if [[ -x "$NOTIFIER" ]]; then
|
||||
"$NOTIFIER" "$text" || true
|
||||
else
|
||||
log "$text"
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_origin_main_with_retry() {
|
||||
local -a retry_delays=(0 5 20)
|
||||
local -a git_options=()
|
||||
local attempt delay
|
||||
|
||||
for ((attempt = 1; attempt <= ${#retry_delays[@]}; attempt++)); do
|
||||
delay="${retry_delays[attempt - 1]}"
|
||||
if ((delay > 0)); then
|
||||
log "Hermes-Git-Fetch: neuer Versuch $attempt/${#retry_delays[@]} in ${delay}s."
|
||||
sleep "$delay"
|
||||
fi
|
||||
|
||||
git_options=()
|
||||
if ((attempt == ${#retry_delays[@]})); then
|
||||
git_options=(-c http.version=HTTP/1.1)
|
||||
log "Hermes-Git-Fetch: letzter Versuch mit HTTP/1.1."
|
||||
fi
|
||||
|
||||
if GIT_TERMINAL_PROMPT=0 timeout --foreground 90s \
|
||||
git "${git_options[@]}" -C "$HERMES_REPO" fetch --no-tags \
|
||||
"$HERMES_REMOTE" \
|
||||
"+refs/heads/$HERMES_BRANCH:refs/remotes/$HERMES_REMOTE/$HERMES_BRANCH"; then
|
||||
((attempt == 1)) ||
|
||||
log "Hermes-Git-Fetch war im Versuch $attempt/${#retry_delays[@]} erfolgreich."
|
||||
return 0
|
||||
fi
|
||||
log "Hermes-Git-Fetch fehlgeschlagen (Versuch $attempt/${#retry_delays[@]})."
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
run_hermes_update_attempt() {
|
||||
local attempt="$1"
|
||||
local -a pipe_status
|
||||
|
||||
printf '\n===== Hermes update: Versuch %s/2 (%s) =====\n' \
|
||||
"$attempt" "$(date --iso-8601=seconds)" >>"$update_log"
|
||||
|
||||
set +e
|
||||
hermes update 2>&1 | tee -a "$update_log"
|
||||
pipe_status=("${PIPESTATUS[@]}")
|
||||
set -e
|
||||
|
||||
((pipe_status[1] == 0)) || return "${pipe_status[1]}"
|
||||
return "${pipe_status[0]}"
|
||||
}
|
||||
|
||||
is_retryable_update_fetch_failure() {
|
||||
local current_commit dirty_after
|
||||
current_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)" || return 1
|
||||
[[ "$current_commit" == "$old_commit" ]] || return 1
|
||||
|
||||
dirty_after="$(
|
||||
git -C "$HERMES_REPO" status --porcelain --untracked-files=all |
|
||||
grep -vxF '?? .install_method' || true
|
||||
)"
|
||||
[[ -z "$dirty_after" ]] || return 1
|
||||
grep -Fq 'Fetching updates' "$update_log" || return 1
|
||||
grep -Fq 'Pulling updates' "$update_log" && return 1
|
||||
grep -Eqi \
|
||||
'HTTP (429|500|502|503|504)|returned error: (429|500|502|503|504)|rate.?limit|could not read Username|Authentication failed|Could not resolve host|Failed to connect|Connection (timed out|reset)|TLS|early EOF|Failed to fetch updates|Network error' \
|
||||
"$update_log"
|
||||
}
|
||||
|
||||
for command in git tar zstd jq curl systemctl timeout flock hermes; do
|
||||
command -v "$command" >/dev/null 2>&1 || {
|
||||
notify_user "Hermes-Update abgebrochen: benötigtes Kommando fehlt: $command"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
LOCK_FILE="${XDG_RUNTIME_DIR:-/tmp}/hermes-stack-hermes-update.lock"
|
||||
exec 9>"$LOCK_FILE"
|
||||
flock -n 9 || exit 0
|
||||
|
||||
if [[ ! -d "$HERMES_REPO/.git" ]]; then
|
||||
notify_user "Hermes-Update abgebrochen: Git-Installation nicht gefunden."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$HERMES_REPO" in
|
||||
"$HOME"/*/hermes-agent) ;;
|
||||
*)
|
||||
notify_user "Hermes-Update abgebrochen: unsicherer Repository-Pfad: $HERMES_REPO"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
dirty="$(
|
||||
git -C "$HERMES_REPO" status --porcelain --untracked-files=all |
|
||||
grep -vxF '?? .install_method' || true
|
||||
)"
|
||||
if [[ -n "$dirty" ]]; then
|
||||
notify_user "Hermes-Update abgebrochen: lokale Änderungen im Hermes-Repository:
|
||||
$dirty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
old_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)"
|
||||
|
||||
if ! fetch_origin_main_with_retry; then
|
||||
notify_user "Hermes-Updateprüfung fehlgeschlagen: $HERMES_REMOTE/$HERMES_BRANCH ist nicht erreichbar. Gateway und Installation bleiben unverändert."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
remote_commit="$(git -C "$HERMES_REPO" rev-parse "refs/remotes/$HERMES_REMOTE/$HERMES_BRANCH")"
|
||||
if [[ ! "$remote_commit" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
notify_user "Hermes-Updateprüfung fehlgeschlagen: Remote lieferte keinen gültigen Commit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$remote_commit" == "$old_commit" ]]; then
|
||||
notify_user "Hermes ist bereits aktuell: $old_commit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
backup_dir="$STATE_DIR/backups"
|
||||
mkdir -p "$backup_dir"
|
||||
timestamp="$(date +%Y%m%d-%H%M%S)"
|
||||
backup="$backup_dir/hermes-$timestamp.tar.zst"
|
||||
update_log="$STATE_DIR/hermes-update-$timestamp.log"
|
||||
|
||||
paths=("${HERMES_REPO#"$HOME/"}")
|
||||
for path in \
|
||||
"$HOME/.hermes/config.yaml" "$HOME/.hermes/.env" "$HOME/.hermes/auth.json" \
|
||||
"${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user/$GATEWAY_SERVICE"; do
|
||||
[[ -e "$path" && "$path" == "$HOME/"* ]] && paths+=("${path#"$HOME/"}")
|
||||
done
|
||||
|
||||
tar --zstd -cf "$backup" -C "$HOME" "${paths[@]}"
|
||||
|
||||
rollback() {
|
||||
trap - ERR
|
||||
set +e
|
||||
systemctl --user stop "$GATEWAY_SERVICE"
|
||||
if [[ "$HERMES_REPO" == "$HOME/"*/hermes-agent ]]; then
|
||||
rm -rf -- "$HERMES_REPO"
|
||||
tar --zstd -xf "$backup" -C "$HOME"
|
||||
fi
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user restart "$GATEWAY_SERVICE"
|
||||
notify_user "Hermes-Update zurückgerollt auf $old_commit. Backup: $backup"
|
||||
exit 1
|
||||
}
|
||||
trap rollback ERR
|
||||
|
||||
systemctl --user stop "$GATEWAY_SERVICE"
|
||||
|
||||
if run_hermes_update_attempt 1; then
|
||||
update_exit=0
|
||||
else
|
||||
update_exit=$?
|
||||
log "Hermes-Updater meldete Exitcode $update_exit."
|
||||
if is_retryable_update_fetch_failure; then
|
||||
log "Transienter Fetch-Fehler erkannt; Hermes-Update wird einmal wiederholt."
|
||||
sleep 20
|
||||
if run_hermes_update_attempt 2; then
|
||||
update_exit=0
|
||||
else
|
||||
update_exit=$?
|
||||
log "Hermes-Updater-Retry meldete Exitcode $update_exit."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! hermes config check >/dev/null 2>&1; then
|
||||
hermes config migrate
|
||||
hermes config check
|
||||
fi
|
||||
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user reset-failed "$GATEWAY_SERVICE" || true
|
||||
systemctl --user restart "$GATEWAY_SERVICE"
|
||||
|
||||
gateway_ready=0
|
||||
for ((attempt = 1; attempt <= ${GATEWAY_HEALTH_ATTEMPTS:-12}; attempt++)); do
|
||||
if systemctl --user is-active --quiet "$GATEWAY_SERVICE" &&
|
||||
hermes gateway status >/dev/null 2>&1; then
|
||||
gateway_ready=1
|
||||
break
|
||||
fi
|
||||
sleep "${GATEWAY_HEALTH_DELAY:-5}"
|
||||
done
|
||||
|
||||
if ((gateway_ready == 0)); then
|
||||
notify_user "Hermes-Gateway wurde nach dem Update nicht gesund. Rollback wird ausgeführt."
|
||||
false
|
||||
fi
|
||||
|
||||
if ! curl --fail --silent --max-time 10 "$LLAMA_URL/health" |
|
||||
jq -e '.status == "ok"' >/dev/null; then
|
||||
notify_user "Hermes und Gateway sind aktualisiert, aber der lokale Modell-Endpunkt ist nicht gesund: $LLAMA_URL. Kein Hermes-Rollback, da llama-server unabhängig ist."
|
||||
fi
|
||||
|
||||
trap - ERR
|
||||
|
||||
mapfile -t old_backups < <(
|
||||
find "$backup_dir" -maxdepth 1 -type f -name 'hermes-*.tar.zst' \
|
||||
-printf '%T@ %p\n' | sort -nr | tail -n "+$((${BACKUP_KEEP:-3} + 1))" |
|
||||
cut -d' ' -f2-
|
||||
)
|
||||
((${#old_backups[@]} == 0)) || rm -f -- "${old_backups[@]}"
|
||||
|
||||
new_commit="$(git -C "$HERMES_REPO" rev-parse HEAD)"
|
||||
if [[ "$new_commit" == "$old_commit" ]]; then
|
||||
notify_user "Hermes-Update wurde nicht installiert (Updater-Exitcode $update_exit); der bisherige Commit $new_commit und der frisch gestartete Gateway sind gesund."
|
||||
exit 1
|
||||
elif ((update_exit != 0)); then
|
||||
notify_user "Hermes erfolgreich aktualisiert: $old_commit -> $new_commit. Eine Updater-Restartwarnung wurde durch den frischen Gateway-Neustart behoben."
|
||||
else
|
||||
notify_user "Hermes erfolgreich aktualisiert: $old_commit -> $new_commit"
|
||||
fi
|
||||
Reference in new issue
Block a user