Make local inference limits and workstation setup reproducible

scripts/llama_watchdog.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Bound a busy llama.cpp slot even if the client loses its cancellation path.
+
+Only task IDs, monotonic timestamps and service invocation IDs are persisted.
+The wrapper holds the same maintenance lock as the updater and restart helper.
+"""
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+from urllib.request import urlopen
+
+
+def observe(previous, slots, invocation, now, seconds, tokens):
+    old = previous if previous.get("invocation") == invocation else {}
+    active = {}
+    reasons = []
+    for slot in slots:
+        if not slot.get("is_processing"):
+            continue
+        if "id_task" not in slot:
+            raise ValueError("busy slot without task ID")
+        key = f'{slot["id"]}:{slot["id_task"]}'
+        started = old.get("active", {}).get(key, now)
+        if started > now:
+            started = now
+        active[key] = started
+        decoded = max((item.get("n_decoded", 0) for item in slot.get("next_token", [])), default=0)
+        if now - started >= seconds:
+            reasons.append(f"slot {slot['id']}: request exceeded {seconds}s")
+        if decoded > tokens:
+            reasons.append(f"slot {slot['id']}: output exceeded {tokens} tokens")
+    return {"invocation": invocation, "active": active}, reasons
+
+
+def systemctl(*args):
+    return subprocess.run(["systemctl", "--user", *args], check=True,
+                          capture_output=True, text=True, timeout=35).stdout.strip()
+
+
+def main():
+    service = os.environ.get("LLAMA_SERVICE", "llama-qwen.service")
+    url = os.environ.get("LLAMA_URL", "http://127.0.0.1:8080").rstrip("/")
+    seconds = int(os.environ.get("LLAMA_MAX_REQUEST_SECONDS", "600"))
+    tokens = int(os.environ.get("LLAMA_MAX_OUTPUT_TOKENS", "2048"))
+    if seconds <= 0 or tokens <= 0:
+        raise ValueError("watchdog limits must be positive")
+    state_dir = Path(os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local/state"))) / "hermes-stack-maintenance"
+    state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
+    path = state_dir / "llama-watchdog.json"
+    try:
+        previous = json.loads(path.read_text())
+    except (OSError, ValueError):
+        previous = {}
+    boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
+    if previous.get("boot") != boot:
+        previous = {}
+    status = systemctl("show", service, "--property=ActiveState", "--property=InvocationID")
+    info = dict(line.split("=", 1) for line in status.splitlines())
+    if info.get("ActiveState") != "active":
+        return  # Respect explicit stops and maintenance; never start an inactive server.
+    invocation = info["InvocationID"]
+    now = time.monotonic()
+    try:
+        with urlopen(url + "/slots", timeout=5) as response:
+            slots = json.load(response)
+        if not isinstance(slots, list):
+            raise ValueError("invalid slot response")
+        state, reasons = observe(previous, slots, invocation, now, seconds, tokens)
+    except (OSError, ValueError, KeyError, TypeError):
+        # A single failed probe is not evidence of a hung model. Require a full
+        # request budget of consecutive failures within the same server process.
+        since = previous.get("unreachable_since", now) if previous.get("invocation") == invocation else now
+        state = {"invocation": invocation, "active": previous.get("active", {}), "unreachable_since": since}
+        reasons = [f"slot endpoint unreachable for {seconds}s"] if now - since >= seconds else []
+    last_restart = previous.get("last_restart", -1e12)
+    state["last_restart"] = last_restart
+    if reasons and now - last_restart >= max(300, seconds):
+        print("llama watchdog: " + "; ".join(reasons) + "; restarting " + service, flush=True)
+        systemctl("restart", "--no-block", service)
+        state = {"invocation": invocation, "active": {}, "last_restart": now}
+    state["boot"] = boot
+    pending = path.with_suffix(".pending")
+    pending.write_text(json.dumps(state) + "\n")
+    pending.chmod(0o600)
+    pending.replace(path)
+
+
+if __name__ == "__main__":
+    main()

scripts/prepare-workstation.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Prepare the documented local model setup; never start services or send messages."""
+import argparse
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import shutil
+import subprocess
+import tempfile
+import time
+
+ROOT = Path(__file__).resolve().parents[1]
+ALIASES = ["hermes-local", "qwen35-4b-local", "qwen38-27b-local", "qwen36-27b-local"]
+LOCAL_URL = "http://127.0.0.1:8080/v1"
+
+
+def verify(path, item):
+    if not path.is_file() or path.stat().st_size != item["size"]:
+        return False
+    with path.open("rb") as model:
+        return hashlib.file_digest(model, "sha256").hexdigest() == item["sha256"]
+
+
+def ensure_model(group, model_dir, download=False):
+    directory = model_dir / group["directory"]
+    for item in group["files"]:
+        path = directory / item["name"]
+        if path.exists():
+            if not verify(path, item):
+                raise ValueError(f"Prüfsumme falsch: {path}; vorhandene Datei bleibt unverändert")
+        else:
+            if not download:
+                raise FileNotFoundError(f"Modell fehlt: {path}; mit --download bereitstellen")
+            directory.mkdir(parents=True, exist_ok=True)
+            partial = path.with_name(path.name + ".part")
+            url = f'https://huggingface.co/{group["repo"]}/resolve/{group["revision"]}/{item["name"]}'
+            subprocess.run(["curl", "-fL", "--retry", "3", "--connect-timeout", "15",
+                            "--max-time", "14400", "--continue-at", "-", url,
+                            "--output", str(partial)], check=True)
+            if not verify(partial, item):
+                raise ValueError(f"Download-Prüfsumme falsch: {partial}; nicht aktiviert")
+            partial.replace(path)
+        print(f"SHA256 OK: {path.name}", flush=True)
+    return [directory / item["name"] for item in group["files"]]
+
+
+def write_config(path, text, replace=False):
+    if path.exists():
+        if path.read_text() == text:
+            return
+        if not replace:
+            print(f"Vorhandene Konfiguration beibehalten: {path}")
+            return
+        backup = path.with_name(path.name + f".before-setup-{time.time_ns()}")
+        shutil.copy2(path, backup)
+        backup.chmod(0o600)
+        print(f"Sicherung: {backup}")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as pending:
+        pending.write(text)
+        staging = Path(pending.name)
+    staging.chmod(0o600)
+    staging.replace(path)
+
+
+def profile_config(name, paths, server_bin=""):
+    args = ["--ctx-size", "65536", "--parallel", "1", "--jinja"]
+    if name == "compression":
+        args += ["--rope-scaling", "yarn", "--rope-scale", "2", "--yarn-orig-ctx", "32768",
+                 "--reasoning-budget", "0", "--no-reasoning-preserve", "--n-gpu-layers", "0",
+                 "--device", "none", "--no-op-offload", "--no-kv-offload",
+                 "--threads", "8", "--threads-batch", "8"]
+    else:
+        args += ["--reasoning", "auto", "--reasoning-budget", "128", "--no-reasoning-preserve",
+                 "--threads", "6", "--threads-batch", "16"]
+        if name == "routine":
+            # Fits on the verified 10-GiB workstation. Explicit layers disable fit.
+            args += ["--n-gpu-layers", "99", "--fit", "off"]
+        else:
+            args += ["--n-gpu-layers", "auto", "--fit", "on", "--fit-target", "4096"]
+        args += ["--sleep-idle-seconds", "300", "--image-min-tokens", "1024",
+                 "--cors-origins", "localhost"]
+    args += ["--flash-attn", "auto", "--cache-type-k", "q8_0", "--cache-type-v", "q8_0", "--no-ui"]
+    settings = {"LLAMA_SERVER_BIN": server_bin, "LLAMA_MODEL": str(paths[0]),
+                "LLAMA_MMPROJ": str(paths[1]) if len(paths) > 1 else "", "LLAMA_HF_REPO": "",
+                "LLAMA_ALIAS": "qwen3-1.7b-compression" if name == "compression" else ",".join(ALIASES),
+                "LLAMA_HOST": "127.0.0.1", "LLAMA_PORT": "8081" if name == "compression" else "8080",
+                "LLAMA_PROFILE": "", "LLAMA_MAX_OUTPUT_TOKENS": "2048"}
+    return ("# Generated from config/models.lock.json; no service started.\n"
+            + "".join(f"{key}={shlex.quote(value)}\n" for key, value in settings.items())
+            + "LLAMA_ARGS=(\n  " + shlex.join(args) + "\n)\n")
+
+
+def configure_hermes(config, compression=False):
+    config.setdefault("model", {}).update(default="hermes-local", provider="custom", base_url=LOCAL_URL)
+    config.setdefault("agent", {}).update(run_budget_seconds=1800, budget_warning_ratio=0.8)
+    config.setdefault("providers", {}).setdefault("custom", {}).update(
+        request_timeout_seconds=600, stale_timeout_seconds=180)
+    entries = config.setdefault("custom_providers", [])
+    matches = [entry for entry in entries if entry.get("base_url", "").rstrip("/") == LOCAL_URL]
+    if not matches:
+        matches = [{"name": "Local (127.0.0.1:8080)", "base_url": LOCAL_URL}]
+        entries.extend(matches)
+    for entry in matches:
+        entry["model"] = "hermes-local"
+        entry.setdefault("models", {}).update({name: {} for name in ALIASES})
+        entry.setdefault("extra_body", {}).update(max_tokens=2048, t_max_predict_ms=540000)
+    auxiliary = config.setdefault("auxiliary", {})
+    auxiliary.setdefault("vision", {}).update(provider="main", model="hermes-local",
+        base_url=LOCAL_URL, api_key="local-no-key", timeout=300, reasoning_effort="none", max_concurrency=1)
+    if compression:
+        auxiliary.setdefault("compression", {}).update(provider="custom", model="qwen3-1.7b-compression",
+            base_url="http://127.0.0.1:8081/v1", api_key="local-no-key", timeout=300, reasoning_effort="none")
+        config.setdefault("compression", {}).update(enabled=True, threshold=0.5,
+            threshold_tokens=24000, target_ratio=0.2, protect_last_n=10, protect_first_n=3,
+            context_timeout_seconds=120, context_total_ceiling_seconds=600)
+    return config
+
+
+def maintenance_config(text):
+    for key, value in {"LLAMA_SERVICE": "llama-qwen.service", "LLAMA_URL": "http://127.0.0.1:8080",
+                       "MODEL_NAME": "hermes-local", "LLAMA_MAX_REQUEST_SECONDS": "600",
+                       "LLAMA_MAX_OUTPUT_TOKENS": "2048"}.items():
+        setting = f"{key}={shlex.quote(value)}"
+        pattern = rf"^{key}=.*$"
+        text = re.sub(pattern, lambda _: setting, text, flags=re.MULTILINE) if re.search(pattern, text, re.MULTILINE) else text + "\n" + setting + "\n"
+    return text
+
+
+def unpin_local_jobs(path):
+    if not path.exists():
+        return
+    for job in json.loads(path.read_text()).get("jobs", []):
+        if (job.get("model") in ALIASES and job.get("provider") == "custom"
+                and job.get("base_url") in (None, "", LOCAL_URL)):
+            subprocess.run(["hermes", "cron", "edit", job["id"], "--unpin"], check=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--download", action="store_true", help="Fehlende, fest versionierte GGUF-Dateien herunterladen")
+    parser.add_argument("--models", default="routine,large,compression", help="Kommagetrennt: routine,large,compression")
+    parser.add_argument("--server-bin", default="", help="Leer: verwalteter CUDA-Build oder llama-server im PATH")
+    parser.add_argument("--model-dir", type=Path, default=Path.home() / ".local/share/hermes-stack/models")
+    parser.add_argument("--config-dir", type=Path, default=Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) / "hermes-stack-maintenance")
+    parser.add_argument("--hermes-config", type=Path, default=Path(os.environ.get("HERMES_HOME", str(Path.home() / ".hermes"))) / "config.yaml")
+    parser.add_argument("--select", choices=["routine", "large"], help="Aktive llama.env vorbereiten; kein Dienststart")
+    parser.add_argument("--configure-hermes", action="store_true", help="Gezielte Hermes-/Maintenance-Einstellungen mit Backup übernehmen")
+    parser.add_argument("--unpin-local-cron", action="store_true", help="Lokale Cron-Modellbindungen über Hermes-CLI lösen")
+    options = parser.parse_args()
+    names = list(dict.fromkeys(options.models.split(",")))
+    manifest = json.loads((ROOT / "config/models.lock.json").read_text())["models"]
+    if not names or any(name not in manifest for name in names):
+        parser.error("--models erlaubt nur routine,large,compression")
+    if options.select and options.select not in names:
+        parser.error("--select muss in --models enthalten sein")
+    if options.configure_hermes and "routine" not in names and "large" not in names:
+        parser.error("--configure-hermes benötigt ein Hauptmodell")
+    if options.select or options.configure_hermes or options.unpin_local_cron:
+        state = subprocess.check_output(["systemctl", "--user", "show", "hermes-gateway.service",
+            "--property=ActiveState", "--value"], text=True).strip()
+        if state not in ("inactive", "failed", ""):
+            parser.error("Gateway zuerst mit systemctl --user stop hermes-gateway.service anhalten")
+    # Import before downloads, so a missing dependency does not waste bandwidth.
+    if options.configure_hermes:
+        import yaml
+    lock_path = Path(os.environ.get("XDG_RUNTIME_DIR", "/tmp")) / "hermes-stack-hermes-update.lock"
+    with lock_path.open("w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        profiles = {}
+        for name in names:
+            paths = ensure_model(manifest[name], options.model_dir, options.download)
+            profiles[name] = profile_config(name, paths, options.server_bin)
+        for name, text in profiles.items():
+            filename = "compression.env" if name == "compression" else f"llama-{name}.env"
+            write_config(options.config_dir / filename, text)
+        if options.select:
+            source = options.config_dir / f"llama-{options.select}.env"
+            write_config(options.config_dir / "llama.env", source.read_text(), replace=True)
+        if options.configure_hermes:
+            text = options.hermes_config.read_text() if options.hermes_config.exists() else ""
+            config = yaml.safe_load(text) or {}
+            configure_hermes(config, "compression" in names)
+            write_config(options.hermes_config, yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=1000), replace=True)
+            maintenance = options.config_dir / "config.env"
+            text = maintenance.read_text() if maintenance.exists() else (ROOT / "config/config.env.example").read_text()
+            write_config(maintenance, maintenance_config(text), replace=True)
+        if options.unpin_local_cron:
+            unpin_local_jobs(options.hermes_config.parent / "cron/jobs.json")
+    print("Vorbereitung abgeschlossen. Keine Dienste gestartet. Siehe docs/NEW_WORKSTATION.md.")
+
+
+if __name__ == "__main__":
+    main()