108 lines
3.6 KiB
Bash
Executable File
108 lines
3.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
config_dir="${XDG_CONFIG_HOME:-$HOME/.config}/hermes-stack-maintenance"
|
|
CONFIG_FILE="${HERMES_LLAMA_CONFIG:-$config_dir/llama.env}"
|
|
LLAMA_ARGS=()
|
|
# shellcheck disable=SC1090
|
|
source "$CONFIG_FILE"
|
|
|
|
mode=run
|
|
profile_override=""
|
|
while (($#)); do
|
|
case "$1" in
|
|
--check) mode=check ;;
|
|
--print) mode=print ;;
|
|
--profile) [[ $# -ge 2 ]] || exit 2; profile_override="$2"; shift ;;
|
|
*) printf 'Aufruf: %s [--check|--print] [--profile NAME]\n' "$0" >&2; exit 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
if [[ -n "${LLAMA_PROFILE:-}" && -r "$config_dir/llama-profile.env" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "$config_dir/llama-profile.env"
|
|
fi
|
|
[[ -z "$profile_override" ]] || LLAMA_PROFILE="$profile_override"
|
|
|
|
if [[ -z "${LLAMA_SERVER_BIN:-}" ]]; then
|
|
LLAMA_SERVER_BIN="$HOME/.local/opt/llama.cpp/current/bin/llama-server"
|
|
if [[ ! -x "$LLAMA_SERVER_BIN" ]]; then
|
|
LLAMA_SERVER_BIN="$(command -v llama-server || true)"
|
|
fi
|
|
fi
|
|
[[ -x "$LLAMA_SERVER_BIN" ]] || {
|
|
printf 'llama-server nicht ausführbar: %s\n' "$LLAMA_SERVER_BIN" >&2
|
|
exit 1
|
|
}
|
|
|
|
model_args=()
|
|
if [[ -n "${LLAMA_MODEL:-}" && -n "${LLAMA_HF_REPO:-}" ]]; then
|
|
printf 'Nur LLAMA_MODEL ODER LLAMA_HF_REPO setzen.\n' >&2
|
|
exit 1
|
|
elif [[ -n "${LLAMA_MODEL:-}" ]]; then
|
|
[[ -r "$LLAMA_MODEL" ]] || { printf 'Modell nicht lesbar.\n' >&2; exit 1; }
|
|
model_args=(--model "$LLAMA_MODEL")
|
|
elif [[ -n "${LLAMA_HF_REPO:-}" ]]; then
|
|
model_args=(--hf-repo "$LLAMA_HF_REPO")
|
|
else
|
|
printf 'LLAMA_MODEL oder LLAMA_HF_REPO fehlt.\n' >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "${LLAMA_MMPROJ:-}" ]]; then
|
|
[[ -r "$LLAMA_MMPROJ" ]] || { printf 'Projektor nicht lesbar.\n' >&2; exit 1; }
|
|
model_args+=(--mmproj "$LLAMA_MMPROJ")
|
|
fi
|
|
[[ -z "${LLAMA_ALIAS:-}" ]] || model_args+=(--alias "$LLAMA_ALIAS")
|
|
|
|
profile_args=()
|
|
if [[ -n "${LLAMA_PROFILE:-}" ]]; then
|
|
case "$LLAMA_PROFILE" in
|
|
baseline) ctx=65536; margin=1536 ;;
|
|
balanced) ctx=49152; margin=1536 ;;
|
|
vram-plus) ctx=49152; margin=1280 ;;
|
|
compact) ctx=32768; margin=1536 ;;
|
|
*) printf 'Unbekanntes Profil: %s\n' "$LLAMA_PROFILE" >&2; exit 2 ;;
|
|
esac
|
|
profile_args=(
|
|
--ctx-size "$ctx" --parallel 1 --jinja
|
|
--reasoning auto --reasoning-budget 512 --reasoning-preserve
|
|
--flash-attn auto --threads 6 --threads-batch 16
|
|
--n-gpu-layers auto --fit on --fit-target "$margin"
|
|
--cache-type-k q8_0 --cache-type-v q8_0
|
|
--sleep-idle-seconds 300 --image-min-tokens 1024
|
|
--cors-origins localhost --no-ui
|
|
)
|
|
# Alte vollständige LLAMA_ARGS nicht still mit einem Profil vermischen.
|
|
if ((${#LLAMA_ARGS[@]})); then
|
|
printf 'Bei Profilbetrieb LLAMA_ARGS leeren; eigene vollständige Args nur ohne LLAMA_PROFILE.\n' >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
args=("${model_args[@]}" --host "${LLAMA_HOST:-127.0.0.1}" --port "${LLAMA_PORT:-8080}"
|
|
"${profile_args[@]}" "${LLAMA_ARGS[@]}")
|
|
if [[ "$mode" == print ]]; then
|
|
printf '%q ' "$LLAMA_SERVER_BIN" "${args[@]}"
|
|
printf '\n'
|
|
exit 0
|
|
fi
|
|
|
|
# Kein Download/Modellstart: nur prüfen, ob die installierte Version die Namen kennt.
|
|
help_text="$(timeout 30 "$LLAMA_SERVER_BIN" --help 2>&1)" || {
|
|
printf 'llama-server --help fehlgeschlagen.\n' >&2; exit 1;
|
|
}
|
|
for arg in "${args[@]}"; do
|
|
[[ "$arg" == --* ]] || continue
|
|
option="${arg%%=*}"
|
|
if ! grep -Eq -- "(^|[[:space:],])${option}([[:space:],=]|$)" <<<"$help_text"; then
|
|
printf 'Die installierte llama.cpp-Version kennt %s nicht. Kein automatisches Weglassen.\n' "$option" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ "$mode" == check ]]; then
|
|
printf 'Konfiguration und Optionsnamen OK; GPU/Modell/Optionswerte noch nicht laufzeitgetestet.\n'
|
|
exit 0
|
|
fi
|
|
|
|
exec "$LLAMA_SERVER_BIN" "${args[@]}"
|