Hermes-agent

tests/hermes_cli/test_model_catalog.py

@@ -0,0 +1,479 @@
+"""Tests for hermes_cli.model_catalog — remote manifest fetch + cache + fallback."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+
+@pytest.fixture
+def isolated_home(tmp_path, monkeypatch):
+    """Isolate HERMES_HOME + reset any module-level catalog cache per test."""
+    home = tmp_path / ".hermes"
+    home.mkdir()
+    monkeypatch.setattr(Path, "home", lambda: tmp_path)
+    monkeypatch.setenv("HERMES_HOME", str(home))
+
+    # Force a fresh catalog module state for each test.
+    import importlib
+    from hermes_cli import model_catalog
+    importlib.reload(model_catalog)
+    yield home
+    model_catalog.reset_cache()
+
+
+def _valid_manifest() -> dict:
+    return {
+        "version": 1,
+        "updated_at": "2026-04-25T22:00:00Z",
+        "metadata": {"source": "test"},
+        "providers": {
+            "openrouter": {
+                "metadata": {"display_name": "OpenRouter"},
+                "models": [
+                    {"id": "anthropic/claude-opus-4.7", "description": "recommended"},
+                    {"id": "openai/gpt-5.4", "description": ""},
+                    {"id": "openrouter/elephant-alpha", "description": "free"},
+                ],
+            },
+            "nous": {
+                "metadata": {"display_name": "Nous Portal"},
+                "models": [
+                    {"id": "anthropic/claude-opus-4.7"},
+                    {"id": "moonshotai/kimi-k2.6"},
+                ],
+            },
+        },
+    }
+
+
+class TestValidation:
+    def test_accepts_well_formed_manifest(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        assert _validate_manifest(_valid_manifest()) is True
+
+    def test_rejects_non_dict(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        assert _validate_manifest("string") is False
+        assert _validate_manifest([]) is False
+        assert _validate_manifest(None) is False
+
+    def test_rejects_missing_version(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        m = _valid_manifest()
+        del m["version"]
+        assert _validate_manifest(m) is False
+
+    def test_rejects_future_version(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        m = _valid_manifest()
+        m["version"] = 999
+        assert _validate_manifest(m) is False
+
+    def test_rejects_missing_providers(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        m = _valid_manifest()
+        del m["providers"]
+        assert _validate_manifest(m) is False
+
+    def test_rejects_malformed_model_entry(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        m = _valid_manifest()
+        m["providers"]["openrouter"]["models"][0] = {"id": ""}  # empty id
+        assert _validate_manifest(m) is False
+
+    def test_rejects_non_string_model_id(self, isolated_home):
+        from hermes_cli.model_catalog import _validate_manifest
+        m = _valid_manifest()
+        m["providers"]["openrouter"]["models"][0] = {"id": 42}
+        assert _validate_manifest(m) is False
+
+
+class TestFetchSuccess:
+    def test_fetch_and_cache_writes_disk(self, isolated_home):
+        from hermes_cli import model_catalog
+        manifest = _valid_manifest()
+        with patch.object(
+            model_catalog, "_fetch_manifest", return_value=manifest
+        ) as fetch:
+            result = model_catalog.get_catalog(force_refresh=True)
+
+        assert result == manifest
+        assert fetch.called
+
+        cache_file = model_catalog._cache_path()
+        assert cache_file.exists()
+        with open(cache_file) as fh:
+            assert json.load(fh) == manifest
+
+    def test_second_call_uses_in_process_cache(self, isolated_home):
+        from hermes_cli import model_catalog
+        manifest = _valid_manifest()
+        with patch.object(
+            model_catalog, "_fetch_manifest", return_value=manifest
+        ) as fetch:
+            model_catalog.get_catalog(force_refresh=True)
+            model_catalog.get_catalog()  # should not hit network again
+        assert fetch.call_count == 1
+
+    def test_force_refresh_always_refetches(self, isolated_home):
+        from hermes_cli import model_catalog
+        manifest = _valid_manifest()
+        with patch.object(
+            model_catalog, "_fetch_manifest", return_value=manifest
+        ) as fetch:
+            model_catalog.get_catalog(force_refresh=True)
+            model_catalog.get_catalog(force_refresh=True)
+        assert fetch.call_count == 2
+
+
+class TestFetchFailure:
+    def test_network_failure_returns_empty_when_no_cache(self, isolated_home):
+        from hermes_cli import model_catalog
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None):
+            result = model_catalog.get_catalog(force_refresh=True)
+        assert result == {}
+
+    def test_network_failure_falls_back_to_disk_cache(self, isolated_home):
+        from hermes_cli import model_catalog
+        # Prime disk cache with a fresh copy.
+        manifest = _valid_manifest()
+        with patch.object(model_catalog, "_fetch_manifest", return_value=manifest):
+            model_catalog.get_catalog(force_refresh=True)
+
+        # Now wipe in-process cache and simulate network failure on refetch.
+        model_catalog.reset_cache()
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None):
+            result = model_catalog.get_catalog(force_refresh=True)
+
+        assert result == manifest
+
+    def test_fetch_failure_falls_back_to_stale_cache(self, isolated_home):
+        from hermes_cli import model_catalog
+        manifest = _valid_manifest()
+        # Write stale cache directly (mtime in the past).
+        cache = model_catalog._cache_path()
+        cache.parent.mkdir(parents=True, exist_ok=True)
+        with open(cache, "w") as fh:
+            json.dump(manifest, fh)
+        old = time.time() - 30 * 24 * 3600  # 30 days ago
+        import os as _os
+        _os.utime(cache, (old, old))
+
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None):
+            result = model_catalog.get_catalog()
+
+        # Stale cache is better than nothing.
+        assert result == manifest
+
+
+class TestFallbackChain:
+    """``_fetch_manifest_with_fallback`` walks ``DEFAULT_CATALOG_FALLBACK_URLS``
+    when the primary URL fails. Regression: the Docusaurus site behind Vercel
+    occasionally returns HTTP 403 + x-vercel-mitigated: challenge for urllib;
+    without a fallback URL the user's disk cache freezes and new model
+    releases (opus 4.8, etc.) never reach the picker.
+    """
+
+    PRIMARY = "https://hermes-agent.nousresearch.com/docs/api/model-catalog.json"
+    FALLBACK = (
+        "https://raw.githubusercontent.com/NousResearch/hermes-agent"
+        "/main/website/static/api/model-catalog.json"
+    )
+
+    def test_uses_primary_when_it_succeeds(self, isolated_home):
+        from hermes_cli import model_catalog
+        calls: list[str] = []
+
+        def fake_fetch(url, timeout):
+            calls.append(url)
+            return _valid_manifest()
+
+        with patch.object(model_catalog, "_fetch_manifest", side_effect=fake_fetch):
+            result = model_catalog._fetch_manifest_with_fallback(self.PRIMARY, 5.0)
+
+        assert result is not None
+        assert calls == [self.PRIMARY], "fallback URLs must not be touched on primary success"
+
+    def test_falls_through_to_raw_github_on_primary_failure(self, isolated_home):
+        from hermes_cli import model_catalog
+        calls: list[str] = []
+
+        def fake_fetch(url, timeout):
+            calls.append(url)
+            if url == self.PRIMARY:
+                return None  # simulate Vercel 403
+            return _valid_manifest()
+
+        with patch.object(model_catalog, "_fetch_manifest", side_effect=fake_fetch):
+            result = model_catalog._fetch_manifest_with_fallback(self.PRIMARY, 5.0)
+
+        assert result is not None
+        assert calls == [self.PRIMARY, self.FALLBACK]
+
+    def test_returns_none_when_all_urls_fail(self, isolated_home):
+        from hermes_cli import model_catalog
+
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None) as fetch:
+            result = model_catalog._fetch_manifest_with_fallback(self.PRIMARY, 5.0)
+
+        assert result is None
+        # Primary + every fallback URL was attempted exactly once.
+        assert fetch.call_count == 1 + len(model_catalog.DEFAULT_CATALOG_FALLBACK_URLS)
+
+    def test_dedupes_when_primary_equals_fallback(self, isolated_home):
+        """Operator who configured ``model_catalog.url`` to the raw GitHub URL
+        should not get a duplicate fetch from the fallback list."""
+        from hermes_cli import model_catalog
+
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None) as fetch:
+            model_catalog._fetch_manifest_with_fallback(self.FALLBACK, 5.0)
+
+        assert fetch.call_count == 1, f"expected 1 call, got {fetch.call_count}"
+
+    def test_get_catalog_uses_fallback_chain(self, isolated_home):
+        """End-to-end: ``get_catalog`` routes through the fallback helper so
+        a primary URL failure transparently produces a working catalog."""
+        from hermes_cli import model_catalog
+        manifest = _valid_manifest()
+        calls: list[str] = []
+
+        def fake_fetch(url, timeout):
+            calls.append(url)
+            if url == self.PRIMARY:
+                return None
+            return manifest
+
+        with patch.object(model_catalog, "_fetch_manifest", side_effect=fake_fetch):
+            result = model_catalog.get_catalog(force_refresh=True)
+
+        assert result == manifest
+        assert self.FALLBACK in calls
+
+
+class TestCuratedAccessors:
+    def test_openrouter_returns_tuples(self, isolated_home):
+        from hermes_cli import model_catalog
+        with patch.object(
+            model_catalog, "_fetch_manifest", return_value=_valid_manifest()
+        ):
+            result = model_catalog.get_curated_openrouter_models()
+        assert result == [
+            ("anthropic/claude-opus-4.7", "recommended"),
+            ("openai/gpt-5.4", ""),
+            ("openrouter/elephant-alpha", "free"),
+        ]
+
+    def test_nous_returns_ids(self, isolated_home):
+        from hermes_cli import model_catalog
+        with patch.object(
+            model_catalog, "_fetch_manifest", return_value=_valid_manifest()
+        ):
+            result = model_catalog.get_curated_nous_models()
+        assert result == ["anthropic/claude-opus-4.7", "moonshotai/kimi-k2.6"]
+
+    def test_openrouter_returns_none_when_catalog_empty(self, isolated_home):
+        from hermes_cli import model_catalog
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None):
+            assert model_catalog.get_curated_openrouter_models() is None
+
+    def test_nous_returns_none_when_catalog_empty(self, isolated_home):
+        from hermes_cli import model_catalog
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None):
+            assert model_catalog.get_curated_nous_models() is None
+
+
+class TestDisabled:
+    def test_disabled_config_short_circuits(self, isolated_home):
+        from hermes_cli import model_catalog
+        with patch.object(
+            model_catalog,
+            "_load_catalog_config",
+            return_value={
+                "enabled": False,
+                "url": "http://ignored",
+                "ttl_hours": 24.0,
+                "providers": {},
+            },
+        ):
+            with patch.object(model_catalog, "_fetch_manifest") as fetch:
+                result = model_catalog.get_catalog()
+        assert result == {}
+        fetch.assert_not_called()
+
+
+class TestProviderOverride:
+    def test_override_url_takes_precedence(self, isolated_home):
+        from hermes_cli import model_catalog
+
+        override_payload = {
+            "version": 1,
+            "providers": {
+                "openrouter": {
+                    "models": [
+                        {"id": "override/model", "description": "custom"},
+                    ]
+                }
+            },
+        }
+
+        def fake_fetch(url, timeout):
+            if "override" in url:
+                return override_payload
+            return _valid_manifest()
+
+        with patch.object(
+            model_catalog,
+            "_load_catalog_config",
+            return_value={
+                "enabled": True,
+                "url": "http://master",
+                "ttl_hours": 24.0,
+                "providers": {"openrouter": {"url": "http://override"}},
+            },
+        ):
+            with patch.object(model_catalog, "_fetch_manifest", side_effect=fake_fetch):
+                result = model_catalog.get_curated_openrouter_models()
+
+        assert result == [("override/model", "custom")]
+
+
+class TestIntegrationWithModelsModule:
+    """Exercise the fallback paths via the real callers in hermes_cli.models."""
+
+    def test_curated_nous_ids_falls_back_to_hardcoded_on_empty_catalog(
+        self, isolated_home
+    ):
+        from hermes_cli import model_catalog
+        from hermes_cli.models import get_curated_nous_model_ids, _PROVIDER_MODELS
+
+        with patch.object(model_catalog, "_fetch_manifest", return_value=None):
+            result = get_curated_nous_model_ids()
+
+        assert result == list(_PROVIDER_MODELS["nous"])
+
+    def test_curated_nous_ids_prefers_manifest(self, isolated_home):
+        from hermes_cli import model_catalog
+        from hermes_cli.models import get_curated_nous_model_ids
+
+        with patch.object(
+            model_catalog, "_fetch_manifest", return_value=_valid_manifest()
+        ):
+            result = get_curated_nous_model_ids()
+
+        assert result == ["anthropic/claude-opus-4.7", "moonshotai/kimi-k2.6"]
+
+    def test_picker_nous_row_uses_curated_list(self, tmp_path, monkeypatch):
+        """The /model picker surfaces the curated ``_PROVIDER_MODELS["nous"]``
+        list in curated order — matching the ``hermes model`` CLI — not the live
+        ``/v1/models`` catalog or the manifest. Portal free/paid recommendations
+        are unioned in when reachable; offline (as here, with the Portal calls
+        stubbed out) it's exactly the curated list.
+        """
+        # We deliberately do NOT use the ``isolated_home`` fixture here:
+        # that fixture monkeypatches ``Path.home`` to ``tmp_path``, which
+        # trips the auth-store seat-belt in ``_auth_file_path()`` because
+        # ``HERMES_HOME / auth.json`` then resolves to the same path the
+        # seat-belt thinks is the "real" user store. Use the autouse
+        # ``_hermetic_environment`` HERMES_HOME directly instead.
+        import importlib
+        from hermes_cli import model_catalog
+        from hermes_cli.models import get_curated_nous_model_ids
+        importlib.reload(model_catalog)
+        try:
+            from hermes_cli.model_switch import list_picker_providers
+
+            active_home = Path(os.environ["HERMES_HOME"])
+            (active_home / "auth.json").write_text(
+                json.dumps(
+                    {
+                        "providers": {"nous": {"access_token": "fake"}},
+                        "credential_pool": {},
+                    }
+                )
+            )
+
+            # Stub the Portal recommendation union so the row is deterministic
+            # (the curated list alone) and never touches the network. ``expected``
+            # is computed from the same source the picker uses internally
+            # (``curated["nous"] = get_curated_nous_model_ids()``), so the test
+            # stays an invariant — it can't rot as the curated/manifest list grows.
+            with patch.object(
+                model_catalog, "_fetch_manifest", return_value=_valid_manifest()
+            ), patch("hermes_cli.models.check_nous_free_tier", return_value=False), patch(
+                "hermes_cli.models.union_with_portal_free_recommendations",
+                side_effect=lambda ids, *a, **k: (ids, {}),
+            ), patch(
+                "hermes_cli.models.union_with_portal_paid_recommendations",
+                side_effect=lambda ids, *a, **k: (ids, {}),
+            ):
+                expected = get_curated_nous_model_ids()
+                picker = list_picker_providers(
+                    current_provider="nous", max_models=99
+                )
+        finally:
+            model_catalog.reset_cache()
+
+        nous_row = next((r for r in picker if r["slug"] == "nous"), None)
+        assert nous_row is not None, "nous row must appear when authed"
+        assert nous_row["models"] == expected
+
+
+# -----------------------------------------------------------------------------
+# Drift guard — prevent the in-repo curated lists from going out of sync with
+# the docs-hosted manifest at website/static/api/model-catalog.json.
+#
+# History: qwen/qwen3.6-plus was added to _PROVIDER_MODELS["nous"] in commit
+# 9dd6e5510 but website/static/api/model-catalog.json was not regenerated for
+# weeks, so free-tier users on a new install fetched a stale manifest and the
+# free-tier picker showed "No free models currently available." even though
+# the Portal was serving qwen/qwen3.6-plus as free. CI must catch this.
+# -----------------------------------------------------------------------------
+
+
+class TestManifestMatchesInRepoLists:
+    """Fail if the on-disk manifest is out of date relative to in-repo lists."""
+
+    @staticmethod
+    def _strip_volatile(catalog: dict) -> dict:
+        """Drop fields that always change (timestamps) for diff comparison."""
+        out = dict(catalog)
+        out.pop("updated_at", None)
+        return out
+
+    def test_in_repo_lists_match_manifest(self):
+        """``scripts/build_model_catalog.py`` output must match the committed file.
+
+        If this fails, run ``python scripts/build_model_catalog.py`` and
+        commit the regenerated ``website/static/api/model-catalog.json``.
+        """
+        # Resolve the repo root from this test file's location.
+        repo_root = Path(__file__).resolve().parents[2]
+        manifest_path = repo_root / "website" / "static" / "api" / "model-catalog.json"
+
+        if not manifest_path.exists():
+            pytest.skip(f"manifest missing at {manifest_path}")
+
+        # Build expected catalog using the same script CI would.
+        import importlib.util
+        script_path = repo_root / "scripts" / "build_model_catalog.py"
+        spec = importlib.util.spec_from_file_location("_build_model_catalog", script_path)
+        mod = importlib.util.module_from_spec(spec)
+        assert spec.loader is not None
+        spec.loader.exec_module(mod)
+        expected = mod.build_catalog()
+
+        with open(manifest_path, encoding="utf-8") as fh:
+            actual = json.load(fh)
+
+        assert self._strip_volatile(actual) == self._strip_volatile(expected), (
+            "website/static/api/model-catalog.json is out of sync with "
+            "_PROVIDER_MODELS['nous'] / OPENROUTER_MODELS. "
+            "Run: python scripts/build_model_catalog.py && "
+            "git add website/static/api/model-catalog.json"
+        )