Make local inference limits and workstation setup reproducible

tests/smoke.sh

@@ -21,6 +21,8 @@ else
 fi
 
 python3 "$ROOT_DIR/tests/test_llama.py" || failures=$((failures + 1))
+python3 "$ROOT_DIR/tests/test_llama_watchdog.py" || failures=$((failures + 1))
+python3 "$ROOT_DIR/tests/test_workstation.py" || failures=$((failures + 1))
 python3 "$ROOT_DIR/tests/test_hermes_update.py" || failures=$((failures + 1))
 
 if (cd -- "$ROOT_DIR" && rg -n --hidden --glob '!tests/smoke.sh' --glob '!tests/test_llama.py' --glob '!.git/**' \

tests/test_llama.py

@@ -16,7 +16,7 @@ OPTIONS = """--model --hf-repo --mmproj --alias --host --port --ctx-size --paral
 --jinja --reasoning --reasoning-budget --reasoning-preserve --no-reasoning-preserve --flash-attn --threads
 --threads-batch --n-gpu-layers --fit --fit-target --cache-type-k --cache-type-v
 --sleep-idle-seconds --image-min-tokens --cors-origins --no-ui --rope-scaling --rope-scale --yarn-orig-ctx
---device --no-op-offload --no-kv-offload"""
+--device --no-op-offload --no-kv-offload --n-predict"""
 
 
 class LlamaTests(unittest.TestCase):

tests/test_llama_watchdog.py

@@ -0,0 +1,118 @@
+import importlib.util
+import io
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("watchdog", Path(__file__).resolve().parents[1] / "scripts/llama_watchdog.py")
+watchdog = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(watchdog)
+
+
+def busy(task=1, decoded=20):
+    return [{"id": 0, "id_task": task, "is_processing": True,
+             "next_token": [{"n_decoded": decoded}]}]
+
+
+class WatchdogTests(unittest.TestCase):
+    def test_busy_request_is_allowed_then_expires_despite_progress(self):
+        state, reasons = watchdog.observe({}, busy(), "server1", 100, 600, 2048)
+        self.assertFalse(reasons)
+        state, reasons = watchdog.observe(state, busy(decoded=1000), "server1", 699, 600, 2048)
+        self.assertFalse(reasons)
+        _, reasons = watchdog.observe(state, busy(decoded=1100), "server1", 700, 600, 2048)
+        self.assertIn("600s", reasons[0])
+
+    def test_new_request_and_restarted_server_do_not_inherit_deadline(self):
+        state, _ = watchdog.observe({}, busy(), "server1", 100, 600, 2048)
+        for slots, invocation in [(busy(2), "server1"), (busy(), "server2")]:
+            _, reasons = watchdog.observe(state, slots, invocation, 900, 600, 2048)
+            self.assertFalse(reasons)
+
+    def test_idle_slot_drops_old_deadline(self):
+        state, _ = watchdog.observe({}, busy(), "server1", 100, 600, 2048)
+        state, reasons = watchdog.observe(state, [{"id": 0, "is_processing": False}], "server1", 900, 600, 2048)
+        self.assertEqual(state["active"], {})
+        self.assertFalse(reasons)
+
+    def test_excess_output_is_stopped_even_when_client_overrides_default(self):
+        _, reasons = watchdog.observe({}, busy(decoded=2049), "server1", 100, 600, 2048)
+        self.assertIn("2048 tokens", reasons[0])
+
+    def test_prompt_and_generated_text_are_not_persisted(self):
+        slots = busy()
+        slots[0].update(prompt="private prompt", generated="private output")
+        state, _ = watchdog.observe({}, slots, "server1", 100, 600, 2048)
+        self.assertNotIn("private", str(state))
+
+
+class WatchdogIntegrationTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.environment = patch.dict(watchdog.os.environ, {"XDG_STATE_HOME": self.temp.name,
+            "LLAMA_MAX_REQUEST_SECONDS": "600", "LLAMA_MAX_OUTPUT_TOKENS": "2048"})
+        self.environment.start()
+        self.addCleanup(self.environment.stop)
+        self.path = Path(self.temp.name) / "hermes-stack-maintenance/llama-watchdog.json"
+        self.boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
+        self.calls = []
+
+    def control(self, *args):
+        self.calls.append(args)
+        return "ActiveState=active\nInvocationID=test-server" if args[0] == "show" else ""
+
+    def tick(self, now, slots=None, error=False):
+        response = io.BytesIO(json.dumps(slots or []).encode())
+        with patch.object(watchdog, "systemctl", self.control), \
+             patch.object(watchdog.time, "monotonic", return_value=now), \
+             patch.object(watchdog, "urlopen", side_effect=OSError() if error else None, return_value=response):
+            watchdog.main()
+
+    def restarts(self):
+        return [call for call in self.calls if call[0] == "restart"]
+
+    def test_deadline_restarts_actual_configured_service_once(self):
+        self.tick(100, busy())
+        self.tick(699, busy())
+        self.assertFalse(self.restarts())
+        self.tick(700, busy())
+        self.assertEqual(self.restarts(), [("restart", "--no-block", "llama-qwen.service")])
+        self.tick(701, busy(decoded=3000))
+        self.assertEqual(len(self.restarts()), 1)
+        self.assertEqual(self.path.stat().st_mode & 0o777, 0o600)
+
+    def test_only_persistent_probe_failure_triggers_recovery(self):
+        self.tick(100, error=True)
+        self.tick(699, error=True)
+        self.assertFalse(self.restarts())
+        self.tick(700, error=True)
+        self.assertEqual(len(self.restarts()), 1)
+
+    def test_successful_probe_resets_failure_deadline(self):
+        self.tick(100, error=True)
+        self.tick(500)
+        self.tick(700, error=True)
+        self.assertFalse(self.restarts())
+
+    def test_inactive_service_never_started_or_probed(self):
+        with patch.object(watchdog, "systemctl", return_value="ActiveState=inactive\nInvocationID=") as control, \
+             patch.object(watchdog, "urlopen") as probe:
+            watchdog.main()
+        probe.assert_not_called()
+        self.assertEqual(control.call_count, 1)
+
+    def test_reboot_clears_old_monotonic_deadlines_and_cooldown(self):
+        self.path.parent.mkdir()
+        self.path.write_text(json.dumps({"boot": "previous-boot", "invocation": "test-server",
+            "active": {"0:1": 100}, "last_restart": 10000, "unreachable_since": 50}))
+        self.tick(100, busy())
+        self.tick(700, busy())
+        self.assertEqual(len(self.restarts()), 1)
+        self.assertEqual(json.loads(self.path.read_text())["boot"], self.boot)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_workstation.py

@@ -0,0 +1,146 @@
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location("prepare", ROOT / "scripts/prepare-workstation.py")
+prepare = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(prepare)
+
+
+class PreparationTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.data = b"test model weights"
+        self.item = {"name": "weights.gguf", "sha256": hashlib.sha256(self.data).hexdigest(), "size": len(self.data)}
+        self.group = {"repo": "owner/model", "revision": "a" * 40, "directory": "model/revision", "files": [self.item]}
+
+    def test_download_uses_fixed_revision_and_publishes_only_verified_file(self):
+        def download(args, **kwargs):
+            self.assertIn("/resolve/" + "a" * 40 + "/weights.gguf", args[-3])
+            Path(args[-1]).write_bytes(self.data)
+        with patch.object(prepare.subprocess, "run", side_effect=download):
+            paths = prepare.ensure_model(self.group, self.root, True)
+        self.assertEqual(paths[0].read_bytes(), self.data)
+        self.assertFalse(paths[0].with_name("weights.gguf.part").exists())
+
+    def test_corrupt_download_cannot_become_live_model(self):
+        with patch.object(prepare.subprocess, "run", side_effect=lambda args, **_: Path(args[-1]).write_bytes(b"broken")):
+            with self.assertRaisesRegex(ValueError, "nicht aktiviert"):
+                prepare.ensure_model(self.group, self.root, True)
+        self.assertFalse((self.root / "model/revision/weights.gguf").exists())
+
+    def test_existing_wrong_file_is_preserved_without_download(self):
+        path = self.root / "model/revision/weights.gguf"
+        path.parent.mkdir(parents=True)
+        path.write_bytes(b"user file")
+        with patch.object(prepare.subprocess, "run") as download, self.assertRaises(ValueError):
+            prepare.ensure_model(self.group, self.root, True)
+        download.assert_not_called()
+        self.assertEqual(path.read_bytes(), b"user file")
+
+    def test_config_merge_preserves_unrelated_settings_and_external_provider(self):
+        config = {"terminal": {"backend": "docker"}, "plugins": {"enabled": ["private"]},
+                  "custom_providers": [{"name": "Other", "base_url": "https://other.invalid/v1", "model": "other"}]}
+        updated = prepare.configure_hermes(config, True)
+        self.assertEqual(updated["terminal"]["backend"], "docker")
+        self.assertEqual(updated["plugins"]["enabled"], ["private"])
+        self.assertEqual(updated["custom_providers"][0]["model"], "other")
+        self.assertEqual(updated["custom_providers"][1]["extra_body"]["max_tokens"], 2048)
+        self.assertEqual(updated["auxiliary"]["compression"]["base_url"], "http://127.0.0.1:8081/v1")
+
+    def test_existing_profiles_preserved_and_explicit_selection_backed_up(self):
+        path = self.root / "llama.env"
+        path.write_text("custom tuning\n")
+        prepare.write_config(path, "generated\n")
+        self.assertEqual(path.read_text(), "custom tuning\n")
+        prepare.write_config(path, "generated\n", replace=True)
+        prepare.write_config(path, "generated\n", replace=True)
+        self.assertEqual(len(list(self.root.glob("llama.env.before-setup-*"))), 1)
+        self.assertEqual(path.stat().st_mode & 0o777, 0o600)
+
+    def test_profiles_quote_other_user_paths_and_do_not_use_remote_hf_start(self):
+        text = prepare.profile_config("routine", [self.root / "user space/model.gguf", self.root / "projector.gguf"])
+        result = subprocess.run(["bash", "-c", text + '\nprintf "%s|%s|%s" "$LLAMA_MODEL" "$LLAMA_HF_REPO" "${LLAMA_ARGS[*]}"'], text=True, capture_output=True, check=True)
+        self.assertTrue(result.stdout.startswith(str(self.root / "user space/model.gguf") + "||"))
+        self.assertIn("--n-gpu-layers 99 --fit off", result.stdout)
+
+    def test_prepare_whole_new_workstation_offline_without_home_override(self):
+        repository = self.root / "repository"
+        (repository / "config").mkdir(parents=True)
+        shutil.copy2(ROOT / "config/config.env.example", repository / "config")
+        (repository / "config/models.lock.json").write_text(json.dumps({"models": {"routine": self.group}}))
+        path = self.root / "models/model/revision/weights.gguf"
+        path.parent.mkdir(parents=True)
+        path.write_bytes(self.data)
+        config = self.root / "config/hermes-stack-maintenance"
+        hermes = self.root / "hermes/config.yaml"
+        args = ["prepare", "--models", "routine", "--model-dir", str(self.root / "models"),
+                "--config-dir", str(config), "--hermes-config", str(hermes),
+                "--configure-hermes", "--select", "routine"]
+        with patch.object(prepare, "ROOT", repository), patch("sys.argv", args), \
+             patch.dict(os.environ, {"XDG_RUNTIME_DIR": str(self.root)}), \
+             patch.object(prepare.subprocess, "check_output", return_value="inactive\n"):
+            prepare.main()
+        self.assertEqual((config / "llama.env").read_text(), (config / "llama-routine.env").read_text())
+        self.assertIn("LLAMA_MAX_REQUEST_SECONDS=600", (config / "config.env").read_text())
+        self.assertIn("run_budget_seconds: 1800", hermes.read_text())
+
+    def test_local_cron_unpin_does_not_touch_other_provider_jobs(self):
+        path = self.root / "jobs.json"
+        path.write_text(json.dumps({"jobs": [
+            {"id": "local", "model": "qwen38-27b-local", "provider": "custom"},
+            {"id": "cloud", "model": "qwen38-27b-local", "provider": "other"},
+            {"id": "remote", "model": "qwen38-27b-local", "provider": "custom", "base_url": "https://other.invalid/v1"}]}))
+        with patch.object(prepare.subprocess, "run") as edit:
+            prepare.unpin_local_jobs(path)
+        edit.assert_called_once_with(["hermes", "cron", "edit", "local", "--unpin"], check=True)
+
+
+class ModelSwitchTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.bin = self.root / "bin"
+        self.bin.mkdir()
+        self.config = self.root / "config/hermes-stack-maintenance"
+        self.config.mkdir(parents=True)
+        (self.config / "llama.env").write_text("LLAMA_MODEL=old.gguf\n")
+        (self.config / "llama-routine.env").write_text("LLAMA_MODEL=new.gguf\n")
+        shutil.copy2(ROOT / "libexec/hermes-stack-model", self.bin)
+        for name, body in {"hermes-stack-llama-run": "exit 0",
+                           "systemctl": 'echo "$*" >>"$TEST_CALLS"; if [[ "$*" == *"restart llama-qwen"* && "${TEST_FAIL:-0}" == 1 ]]; then exit 1; fi',
+                           "curl": "echo '{\"status\":\"ok\"}'"}.items():
+            p = self.bin / name
+            p.write_text("#!/usr/bin/env bash\n" + body + "\n")
+            p.chmod(0o755)
+        self.env = dict(os.environ, PATH=str(self.bin) + ":" + os.environ["PATH"],
+            XDG_CONFIG_HOME=str(self.config.parent), XDG_RUNTIME_DIR=str(self.root), TEST_CALLS=str(self.root / "calls"))
+
+    def test_successful_switch_through_path_entrypoint(self):
+        (self.root / "entrypoint").symlink_to(self.bin / "hermes-stack-model")
+        result = subprocess.run(["bash", str(self.root / "entrypoint"), "routine"], env=self.env, capture_output=True, text=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual((self.config / "llama.env").read_text(), "LLAMA_MODEL=new.gguf\n")
+        self.assertIn("start hermes-gateway.service", (self.root / "calls").read_text())
+
+    def test_failed_switch_restores_configuration_and_still_attempts_gateway_start(self):
+        result = subprocess.run(["bash", str(self.bin / "hermes-stack-model"), "routine"],
+            env=dict(self.env, TEST_FAIL="1"), capture_output=True, text=True)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual((self.config / "llama.env").read_text(), "LLAMA_MODEL=old.gguf\n")
+        self.assertIn("start hermes-gateway.service", (self.root / "calls").read_text())
+
+
+if __name__ == "__main__":
+    unittest.main()