207 lines
11 KiB
Python
Executable File
207 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Prepare the documented local model setup; never start services or send messages."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
ALIASES = ["hermes-local", "qwen35-9b-local", "qwen35-4b-local", "qwen38-27b-local", "qwen36-27b-local"]
|
|
LOCAL_URL = "http://127.0.0.1:8080/v1"
|
|
|
|
|
|
def verify(path, item):
|
|
if not path.is_file() or path.stat().st_size != item["size"]:
|
|
return False
|
|
with path.open("rb") as model:
|
|
return hashlib.file_digest(model, "sha256").hexdigest() == item["sha256"]
|
|
|
|
|
|
def ensure_model(group, model_dir, download=False):
|
|
directory = model_dir / group["directory"]
|
|
for item in group["files"]:
|
|
path = directory / item["name"]
|
|
if path.exists():
|
|
if not verify(path, item):
|
|
raise ValueError(f"Prüfsumme falsch: {path}; vorhandene Datei bleibt unverändert")
|
|
else:
|
|
if not download:
|
|
raise FileNotFoundError(f"Modell fehlt: {path}; mit --download bereitstellen")
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
partial = path.with_name(path.name + ".part")
|
|
url = f'https://huggingface.co/{group["repo"]}/resolve/{group["revision"]}/{item["name"]}'
|
|
subprocess.run(["curl", "-fL", "--retry", "3", "--connect-timeout", "15",
|
|
"--max-time", "14400", "--continue-at", "-", url,
|
|
"--output", str(partial)], check=True)
|
|
if not verify(partial, item):
|
|
raise ValueError(f"Download-Prüfsumme falsch: {partial}; nicht aktiviert")
|
|
partial.replace(path)
|
|
print(f"SHA256 OK: {path.name}", flush=True)
|
|
return [directory / item["name"] for item in group["files"]]
|
|
|
|
|
|
def write_config(path, text, replace=False):
|
|
if path.exists():
|
|
if path.read_text() == text:
|
|
return
|
|
if not replace:
|
|
print(f"Vorhandene Konfiguration beibehalten: {path}")
|
|
return
|
|
backup = path.with_name(path.name + f".before-setup-{time.time_ns()}")
|
|
shutil.copy2(path, backup)
|
|
backup.chmod(0o600)
|
|
print(f"Sicherung: {backup}")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as pending:
|
|
pending.write(text)
|
|
staging = Path(pending.name)
|
|
staging.chmod(0o600)
|
|
staging.replace(path)
|
|
|
|
|
|
def profile_config(name, paths, server_bin=""):
|
|
context = 65536
|
|
args = ["--ctx-size", str(context), "--parallel", "1", "--jinja"]
|
|
if name == "compression":
|
|
args += ["--rope-scaling", "yarn", "--rope-scale", "2", "--yarn-orig-ctx", "32768",
|
|
"--reasoning-budget", "0", "--no-reasoning-preserve", "--n-gpu-layers", "0",
|
|
"--device", "none", "--no-op-offload", "--no-kv-offload",
|
|
"--threads", "8", "--threads-batch", "8"]
|
|
else:
|
|
args += ["--reasoning", "auto", "--reasoning-budget", "128", "--no-reasoning-preserve",
|
|
"--threads", "6", "--threads-batch", "16"]
|
|
if name in ("routine", "fallback"):
|
|
# Explicit full offload: fail instead of silently moving layers to CPU.
|
|
args += ["--n-gpu-layers", "99", "--fit", "off"]
|
|
else:
|
|
args += ["--n-gpu-layers", "auto", "--fit", "on", "--fit-target", "4096"]
|
|
if name == "routine":
|
|
args += ["--no-mmproj-offload", "--batch-size", "512", "--ubatch-size", "128"]
|
|
args += ["--sleep-idle-seconds", "300", "--image-min-tokens", "1024",
|
|
"--cors-origins", "localhost"]
|
|
cache = "q4_0" if name == "routine" else "q8_0"
|
|
args += ["--flash-attn", "auto", "--cache-type-k", cache, "--cache-type-v", cache, "--no-ui"]
|
|
settings = {"LLAMA_SERVER_BIN": server_bin, "LLAMA_MODEL": str(paths[0]),
|
|
"LLAMA_MMPROJ": str(paths[1]) if len(paths) > 1 else "", "LLAMA_HF_REPO": "",
|
|
"LLAMA_ALIAS": "qwen3-1.7b-compression" if name == "compression" else ",".join(ALIASES),
|
|
"LLAMA_HOST": "127.0.0.1", "LLAMA_PORT": "8081" if name == "compression" else "8080",
|
|
"LLAMA_PROFILE": "", "LLAMA_CONTEXT_TOKENS": str(context), "LLAMA_MAX_OUTPUT_TOKENS": "2048"}
|
|
return ("# Generated from config/models.lock.json; no service started.\n"
|
|
+ "".join(f"{key}={shlex.quote(value)}\n" for key, value in settings.items())
|
|
+ "LLAMA_ARGS=(\n " + shlex.join(args) + "\n)\n")
|
|
|
|
|
|
def configure_hermes(config, compression=False, context=65536):
|
|
if context < 64000:
|
|
raise ValueError("Hermes requires at least 64000 context tokens")
|
|
config.setdefault("model", {}).update(default="hermes-local", provider="custom",
|
|
base_url=LOCAL_URL, context_length=context)
|
|
config.setdefault("agent", {}).update(run_budget_seconds=1800, budget_warning_ratio=0.8)
|
|
config.setdefault("providers", {}).setdefault("custom", {}).update(
|
|
request_timeout_seconds=600, stale_timeout_seconds=180)
|
|
entries = config.setdefault("custom_providers", [])
|
|
matches = [entry for entry in entries if entry.get("base_url", "").rstrip("/") == LOCAL_URL]
|
|
if not matches:
|
|
matches = [{"name": "Local (127.0.0.1:8080)", "base_url": LOCAL_URL}]
|
|
entries.extend(matches)
|
|
for entry in matches:
|
|
entry["model"] = "hermes-local"
|
|
entry.setdefault("models", {}).update({name: {} for name in ALIASES})
|
|
entry.setdefault("extra_body", {}).update(max_tokens=2048, t_max_predict_ms=540000)
|
|
auxiliary = config.setdefault("auxiliary", {})
|
|
auxiliary.setdefault("vision", {}).update(provider="main", model="hermes-local",
|
|
base_url=LOCAL_URL, api_key="local-no-key", timeout=300, reasoning_effort="none", max_concurrency=1)
|
|
if compression:
|
|
auxiliary.setdefault("compression", {}).update(provider="custom", model="qwen3-1.7b-compression",
|
|
base_url="http://127.0.0.1:8081/v1", api_key="local-no-key", timeout=300, reasoning_effort="none")
|
|
config.setdefault("compression", {}).update(enabled=True, threshold=0.5,
|
|
threshold_tokens=24000, target_ratio=0.2, protect_last_n=10, protect_first_n=3,
|
|
context_timeout_seconds=120, context_total_ceiling_seconds=600)
|
|
config.setdefault("compression", {})["threshold_tokens"] = min(24000, context * 3 // 8)
|
|
return config
|
|
|
|
|
|
def maintenance_config(text):
|
|
for key, value in {"LLAMA_SERVICE": "llama-qwen.service", "LLAMA_URL": "http://127.0.0.1:8080",
|
|
"MODEL_NAME": "hermes-local", "LLAMA_MAX_REQUEST_SECONDS": "600",
|
|
"LLAMA_MAX_OUTPUT_TOKENS": "2048"}.items():
|
|
setting = f"{key}={shlex.quote(value)}"
|
|
pattern = rf"^{key}=.*$"
|
|
text = re.sub(pattern, lambda _: setting, text, flags=re.MULTILINE) if re.search(pattern, text, re.MULTILINE) else text + "\n" + setting + "\n"
|
|
return text
|
|
|
|
|
|
def unpin_local_jobs(path):
|
|
if not path.exists():
|
|
return
|
|
for job in json.loads(path.read_text()).get("jobs", []):
|
|
if (job.get("model") in ALIASES and job.get("provider") == "custom"
|
|
and job.get("base_url") in (None, "", LOCAL_URL)):
|
|
subprocess.run(["hermes", "cron", "edit", job["id"], "--unpin"], check=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--download", action="store_true", help="Fehlende, fest versionierte GGUF-Dateien herunterladen")
|
|
parser.add_argument("--models", default="routine,large,compression,fallback", help="Kommagetrennt: routine,large,compression,fallback")
|
|
parser.add_argument("--server-bin", default="", help="Leer: verwalteter CUDA-Build oder llama-server im PATH")
|
|
parser.add_argument("--model-dir", type=Path, default=Path.home() / ".local/share/hermes-stack/models")
|
|
parser.add_argument("--config-dir", type=Path, default=Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) / "hermes-stack-maintenance")
|
|
parser.add_argument("--hermes-config", type=Path, default=Path(os.environ.get("HERMES_HOME", str(Path.home() / ".hermes"))) / "config.yaml")
|
|
parser.add_argument("--select", choices=["routine", "large", "fallback"], help="Aktive llama.env vorbereiten; kein Dienststart")
|
|
parser.add_argument("--configure-hermes", action="store_true", help="Gezielte Hermes-/Maintenance-Einstellungen mit Backup übernehmen")
|
|
parser.add_argument("--unpin-local-cron", action="store_true", help="Lokale Cron-Modellbindungen über Hermes-CLI lösen")
|
|
options = parser.parse_args()
|
|
names = list(dict.fromkeys(options.models.split(",")))
|
|
manifest = json.loads((ROOT / "config/models.lock.json").read_text())["models"]
|
|
if not names or any(name not in manifest for name in names):
|
|
parser.error("--models erlaubt nur routine,large,compression,fallback")
|
|
if options.select and options.select not in names:
|
|
parser.error("--select muss in --models enthalten sein")
|
|
if options.configure_hermes and not any(n in names for n in ("routine", "large", "fallback")):
|
|
parser.error("--configure-hermes benötigt ein Hauptmodell")
|
|
if options.select or options.configure_hermes or options.unpin_local_cron:
|
|
state = subprocess.check_output(["systemctl", "--user", "show", "hermes-gateway.service",
|
|
"--property=ActiveState", "--value"], text=True).strip()
|
|
if state not in ("inactive", "failed", ""):
|
|
parser.error("Gateway zuerst mit systemctl --user stop hermes-gateway.service anhalten")
|
|
# Import before downloads, so a missing dependency does not waste bandwidth.
|
|
if options.configure_hermes:
|
|
import yaml
|
|
lock_path = Path(os.environ.get("XDG_RUNTIME_DIR", "/tmp")) / "hermes-stack-hermes-update.lock"
|
|
with lock_path.open("w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
profiles = {}
|
|
for name in names:
|
|
paths = ensure_model(manifest[name], options.model_dir, options.download)
|
|
profiles[name] = profile_config(name, paths, options.server_bin)
|
|
for name, text in profiles.items():
|
|
filename = "compression.env" if name == "compression" else f"llama-{name}.env"
|
|
write_config(options.config_dir / filename, text)
|
|
if options.select:
|
|
source = options.config_dir / f"llama-{options.select}.env"
|
|
write_config(options.config_dir / "llama.env", source.read_text(), replace=True)
|
|
if options.configure_hermes:
|
|
text = options.hermes_config.read_text() if options.hermes_config.exists() else ""
|
|
config = yaml.safe_load(text) or {}
|
|
configure_hermes(config, "compression" in names, 65536)
|
|
write_config(options.hermes_config, yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=1000), replace=True)
|
|
maintenance = options.config_dir / "config.env"
|
|
text = maintenance.read_text() if maintenance.exists() else (ROOT / "config/config.env.example").read_text()
|
|
write_config(maintenance, maintenance_config(text), replace=True)
|
|
if options.unpin_local_cron:
|
|
unpin_local_jobs(options.hermes_config.parent / "cron/jobs.json")
|
|
print("Vorbereitung abgeschlossen. Keine Dienste gestartet. Siehe docs/NEW_WORKSTATION.md.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|