johannes/hermes-arch-stack
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Trial GPU-resident Qwen3.5-9B with verified 4B rollback

Browse Source
This commit is contained in:
admin committed 2026-10-27 17:14:38 +01:00
1 parent 0929cdb555
commit dec534cc88
10 files changed
+307 -93
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/prepare-workstation.py
+20 -13
View File
@@ -14,7 +14,7 @@ import tempfile
import time
ROOT = Path(__file__).resolve().parents[1]
ALIASES = ["hermes-local", "qwen35-4b-local", "qwen38-27b-local", "qwen36-27b-local"]
ALIASES = ["hermes-local", "qwen35-9b-local", "qwen35-4b-local", "qwen38-27b-local", "qwen36-27b-local"]
LOCAL_URL = "http://127.0.0.1:8080/v1"
@@ -68,7 +68,8 @@ def write_config(path, text, replace=False):
def profile_config(name, paths, server_bin=""):
args = ["--ctx-size", "65536", "--parallel", "1", "--jinja"]
context = 65536
args = ["--ctx-size", str(context), "--parallel", "1", "--jinja"]
if name == "compression":
args += ["--rope-scaling", "yarn", "--rope-scale", "2", "--yarn-orig-ctx", "32768",
"--reasoning-budget", "0", "--no-reasoning-preserve", "--n-gpu-layers", "0",
@@ -77,27 +78,32 @@ def profile_config(name, paths, server_bin=""):
else:
args += ["--reasoning", "auto", "--reasoning-budget", "128", "--no-reasoning-preserve",
"--threads", "6", "--threads-batch", "16"]
if name == "routine":
# Fits on the verified 10-GiB workstation. Explicit layers disable fit.
if name in ("routine", "fallback"):
# Explicit full offload: fail instead of silently moving layers to CPU.
args += ["--n-gpu-layers", "99", "--fit", "off"]
else:
args += ["--n-gpu-layers", "auto", "--fit", "on", "--fit-target", "4096"]
if name == "routine":
args += ["--no-mmproj-offload", "--batch-size", "512", "--ubatch-size", "128"]
args += ["--sleep-idle-seconds", "300", "--image-min-tokens", "1024",
"--cors-origins", "localhost"]
args += ["--flash-attn", "auto", "--cache-type-k", "q8_0", "--cache-type-v", "q8_0", "--no-ui"]
cache = "q4_0" if name == "routine" else "q8_0"
args += ["--flash-attn", "auto", "--cache-type-k", cache, "--cache-type-v", cache, "--no-ui"]
settings = {"LLAMA_SERVER_BIN": server_bin, "LLAMA_MODEL": str(paths[0]),
"LLAMA_MMPROJ": str(paths[1]) if len(paths) > 1 else "", "LLAMA_HF_REPO": "",
"LLAMA_ALIAS": "qwen3-1.7b-compression" if name == "compression" else ",".join(ALIASES),
"LLAMA_HOST": "127.0.0.1", "LLAMA_PORT": "8081" if name == "compression" else "8080",
"LLAMA_PROFILE": "", "LLAMA_MAX_OUTPUT_TOKENS": "2048"}
"LLAMA_PROFILE": "", "LLAMA_CONTEXT_TOKENS": str(context), "LLAMA_MAX_OUTPUT_TOKENS": "2048"}
return ("# Generated from config/models.lock.json; no service started.\n"
+ "".join(f"{key}={shlex.quote(value)}\n" for key, value in settings.items())
+ "LLAMA_ARGS=(\n " + shlex.join(args) + "\n)\n")
def configure_hermes(config, compression=False):
def configure_hermes(config, compression=False, context=65536):
if context < 64000:
raise ValueError("Hermes requires at least 64000 context tokens")
config.setdefault("model", {}).update(default="hermes-local", provider="custom",
base_url=LOCAL_URL, context_length=65536)
base_url=LOCAL_URL, context_length=context)
config.setdefault("agent", {}).update(run_budget_seconds=1800, budget_warning_ratio=0.8)
config.setdefault("providers", {}).setdefault("custom", {}).update(
request_timeout_seconds=600, stale_timeout_seconds=180)
@@ -119,6 +125,7 @@ def configure_hermes(config, compression=False):
config.setdefault("compression", {}).update(enabled=True, threshold=0.5,
threshold_tokens=24000, target_ratio=0.2, protect_last_n=10, protect_first_n=3,
context_timeout_seconds=120, context_total_ceiling_seconds=600)
config.setdefault("compression", {})["threshold_tokens"] = min(24000, context * 3 // 8)
return config
@@ -144,22 +151,22 @@ def unpin_local_jobs(path):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--download", action="store_true", help="Fehlende, fest versionierte GGUF-Dateien herunterladen")
parser.add_argument("--models", default="routine,large,compression", help="Kommagetrennt: routine,large,compression")
parser.add_argument("--models", default="routine,large,compression,fallback", help="Kommagetrennt: routine,large,compression,fallback")
parser.add_argument("--server-bin", default="", help="Leer: verwalteter CUDA-Build oder llama-server im PATH")
parser.add_argument("--model-dir", type=Path, default=Path.home() / ".local/share/hermes-stack/models")
parser.add_argument("--config-dir", type=Path, default=Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) / "hermes-stack-maintenance")
parser.add_argument("--hermes-config", type=Path, default=Path(os.environ.get("HERMES_HOME", str(Path.home() / ".hermes"))) / "config.yaml")
parser.add_argument("--select", choices=["routine", "large"], help="Aktive llama.env vorbereiten; kein Dienststart")
parser.add_argument("--select", choices=["routine", "large", "fallback"], help="Aktive llama.env vorbereiten; kein Dienststart")
parser.add_argument("--configure-hermes", action="store_true", help="Gezielte Hermes-/Maintenance-Einstellungen mit Backup übernehmen")
parser.add_argument("--unpin-local-cron", action="store_true", help="Lokale Cron-Modellbindungen über Hermes-CLI lösen")
options = parser.parse_args()
names = list(dict.fromkeys(options.models.split(",")))
manifest = json.loads((ROOT / "config/models.lock.json").read_text())["models"]
if not names or any(name not in manifest for name in names):
parser.error("--models erlaubt nur routine,large,compression")
parser.error("--models erlaubt nur routine,large,compression,fallback")
if options.select and options.select not in names:
parser.error("--select muss in --models enthalten sein")
if options.configure_hermes and "routine" not in names and "large" not in names:
if options.configure_hermes and not any(n in names for n in ("routine", "large", "fallback")):
parser.error("--configure-hermes benötigt ein Hauptmodell")
if options.select or options.configure_hermes or options.unpin_local_cron:
state = subprocess.check_output(["systemctl", "--user", "show", "hermes-gateway.service",
@@ -185,7 +192,7 @@ def main():
if options.configure_hermes:
text = options.hermes_config.read_text() if options.hermes_config.exists() else ""
config = yaml.safe_load(text) or {}
configure_hermes(config, "compression" in names)
configure_hermes(config, "compression" in names, 65536)
write_config(options.hermes_config, yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=1000), replace=True)
maintenance = options.config_dir / "config.env"
text = maintenance.read_text() if maintenance.exists() else (ROOT / "config/config.env.example").read_text()