Zakaria/hermes-agent
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Hermes-agent

Browse Source
This commit is contained in:
Zakaria
2026-06-14 14:30:48 -04:00
commit dac4b88b94
5058 changed files with 1884848 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/plugins/__init__.py
View File
tests/plugins/browser/__init__.py
View File
tests/plugins/browser/check_parity_vs_main.py
+273
View File
@@ -0,0 +1,273 @@
"""Behavior-parity check for the browser-provider plugin migration (#25214).
Spawns one subprocess per (version, scenario) cell — pinned to either
origin/main (legacy in-tree providers + class-instantiation lookup) or
this PR's worktree (plugin-based registry) via `sys.path[0]`. Each
subprocess clears all browser-related env vars + writes a config.yaml,
loads `tools.browser_tool._get_cloud_provider()`, and emits a reduced
"shape tuple" {is_local, provider_name, is_available} as JSON.
The parent process diffs the shapes per scenario. A diff means the
migration introduced an observable behaviour change vs origin/main —
which would be a real regression for users on the existing config keys.
Run from the PR worktree:
cd ~/.hermes/hermes-agent/.worktrees/browser-providers-plugin
python tests/plugins/browser/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
# Pin one path to current main, one to the PR worktree.
# ``REPO_ROOT`` is ``.../.worktrees/browser-providers-plugin``; the main
# checkout lives two levels up at ``~/.hermes/hermes-agent``.
MAIN_DIR = REPO_ROOT.parent.parent # ~/.hermes/hermes-agent
PR_DIR = REPO_ROOT # the worktree we're in
assert (MAIN_DIR / "tools" / "browser_tool.py").exists(), (
f"MAIN_DIR={MAIN_DIR} doesn't look like a hermes-agent checkout"
)
assert (PR_DIR / "tools" / "browser_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
# Reduced shape comparison — exact instance addresses obviously differ
# between subprocesses, so we compare the parts that matter for users.
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME for the config write.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear every browser-related env var so is_available() is deterministic.
for k in (
"BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", "BROWSERBASE_BASE_URL",
"BROWSER_USE_API_KEY", "BROWSER_USE_GATEWAY_URL",
"FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_BROWSER_TTL",
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
):
os.environ.pop(k, None)
# Apply per-scenario env (passed as JSON via argv[2]).
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
# Apply per-scenario config (passed as YAML body via argv[3]).
config_yaml = sys.argv[3]
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have any browser modules cached.
for name in list(sys.modules):
if name.startswith("tools.") or name.startswith("agent.") or name.startswith("plugins."):
sys.modules.pop(name, None)
from tools.browser_tool import _get_cloud_provider, _is_local_mode
provider = _get_cloud_provider()
# Pull the human-readable backend name via the API that exists on BOTH
# legacy (origin/main: CloudBrowserProvider.provider_name()) and the new
# ABC (BrowserProvider exposes provider_name() as a backward-compat alias
# returning display_name). Both shapes resolve to the same string —
# 'Browserbase' / 'Browser Use' / 'Firecrawl' — so we can compare safely.
provider_name = None
is_available = None
if provider is not None:
pn = getattr(provider, "provider_name", None)
if callable(pn):
provider_name = pn()
elif isinstance(pn, str):
provider_name = pn
is_conf = getattr(provider, "is_configured", None)
if callable(is_conf):
is_available = bool(is_conf())
shape = {
"is_local": _is_local_mode(),
"provider_name": provider_name,
"is_available": is_available,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
# (label, config.yaml body, extra env vars)
("no-config-no-env", "", {}),
("explicit-local-no-env", "browser:\n cloud_provider: local\n", {}),
(
"explicit-browserbase-no-creds",
"browser:\n cloud_provider: browserbase\n",
{},
),
(
"explicit-browserbase-with-creds",
"browser:\n cloud_provider: browserbase\n",
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
),
(
"explicit-browser-use-no-creds",
"browser:\n cloud_provider: browser-use\n",
{},
),
(
"explicit-browser-use-with-creds",
"browser:\n cloud_provider: browser-use\n",
{"BROWSER_USE_API_KEY": "k"},
),
(
"explicit-firecrawl-no-creds",
"browser:\n cloud_provider: firecrawl\n",
{},
),
(
"explicit-firecrawl-with-creds",
"browser:\n cloud_provider: firecrawl\n",
{"FIRECRAWL_API_KEY": "k"},
),
(
"no-config-bu-creds",
"",
{"BROWSER_USE_API_KEY": "k"},
),
(
"no-config-bb-creds",
"",
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
),
(
"no-config-both-creds",
"",
{
"BROWSER_USE_API_KEY": "k",
"BROWSERBASE_API_KEY": "x",
"BROWSERBASE_PROJECT_ID": "y",
},
),
(
"no-config-firecrawl-only",
"",
{"FIRECRAWL_API_KEY": "k"},
),
(
"no-config-firecrawl-and-bb",
"",
{
"FIRECRAWL_API_KEY": "k",
"BROWSERBASE_API_KEY": "x",
"BROWSERBASE_PROJECT_ID": "y",
},
),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
"""Run one (version, scenario) cell. Returns the shape dict."""
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
# Worktrees share the main repo's venv.
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
],
capture_output=True,
text=True,
timeout=30,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout,
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce_for_comparison(shape: dict) -> dict:
"""Reduce a shape dict to the parts that matter for user-visible parity.
We compare ``(is_local, provider_name, is_available)`` — the trio that
decides what the dispatcher does with each tool call. ``provider_name``
is the legacy ``provider_name()`` return value ('Browserbase' / 'Browser
Use' / 'Firecrawl'), which is identical between legacy and plugin
classes (the plugin's ``display_name`` matches the legacy
``provider_name()`` return).
"""
return {
"is_local": shape.get("is_local"),
"provider_name": shape.get("provider_name"),
"is_available": shape.get("is_available"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
failures: list[str] = []
errors: list[str] = []
for label, config_yaml, env in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce_for_comparison(main_shape)
pr_reduced = _reduce_for_comparison(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
tests/plugins/browser/test_browser_provider_plugins.py
+379
View File
@@ -0,0 +1,379 @@
"""Plugin-side tests for the browser provider migration (PR #25214).
Covers:
- All three bundled plugins (browserbase, browser-use, firecrawl)
instantiate and self-report the expected ABC defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The browser_registry resolves an active provider in the documented
scenarios:
* explicit config wins ignoring availability (so dispatcher surfaces
a typed credentials error)
* legacy preference walk: browser-use → browserbase (filtered by
availability)
* firecrawl is NOT in the legacy walk — explicit-only
* unknown name falls through to auto-detect
* ``local`` short-circuits to None
These tests use *real* imports from the plugin modules — no mocking of
provider classes themselves — so the test catches drift in the ABC
interface, the registry, and the plugin glue layer simultaneously.
Mirrors ``tests/plugins/web/test_web_search_provider_plugins.py`` from
PR #25182.
"""
from __future__ import annotations
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _clear_browser_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every browser-provider env var so is_available() returns False."""
for k in (
"BROWSERBASE_API_KEY",
"BROWSERBASE_PROJECT_ID",
"BROWSERBASE_BASE_URL",
"BROWSER_USE_API_KEY",
"BROWSER_USE_GATEWAY_URL",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_BROWSER_TTL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_USER_TOKEN",
):
monkeypatch.delenv(k, raising=False)
def _ensure_plugins_loaded() -> None:
"""Idempotently load plugins so the registry is populated."""
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
# ---------------------------------------------------------------------------
# Per-test isolation
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each test starts with a clean browser-provider env."""
_clear_browser_env(monkeypatch)
# ---------------------------------------------------------------------------
# Bundled plugins register
# ---------------------------------------------------------------------------
class TestBundledPluginsRegister:
"""All three bundled browser plugins discover and register correctly."""
def test_all_three_plugins_present_in_registry(self) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import list_providers
names = sorted(p.name for p in list_providers())
assert names == ["browser-use", "browserbase", "firecrawl"]
@pytest.mark.parametrize(
"plugin_name,expected_display",
[
("browserbase", "Browserbase"),
("browser-use", "Browser Use"),
("firecrawl", "Firecrawl"),
],
)
def test_each_plugin_has_name_and_display_name(
self, plugin_name: str, expected_display: str
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None, f"plugin {plugin_name!r} not registered"
assert provider.name == plugin_name
assert provider.display_name == expected_display
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
"""``get_setup_schema()`` returns a dict the picker can consume."""
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
schema = provider.get_setup_schema()
assert isinstance(schema, dict)
assert "name" in schema
assert "env_vars" in schema
# Every cloud-browser plugin needs the agent-browser post-setup hook
# so the picker auto-installs the CLI on selection.
assert schema.get("post_setup") == "agent_browser"
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_each_plugin_implements_full_lifecycle(self, plugin_name: str) -> None:
"""The ABC's three lifecycle methods are all overridden."""
_ensure_plugins_loaded()
from agent.browser_provider import BrowserProvider
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
# Each method must be a real override, not the ABC's NotImplementedError
# default — we check by comparing the function reference.
assert type(provider).create_session is not BrowserProvider.create_session
assert type(provider).close_session is not BrowserProvider.close_session
assert (
type(provider).emergency_cleanup is not BrowserProvider.emergency_cleanup
)
# ---------------------------------------------------------------------------
# is_available() behavior
# ---------------------------------------------------------------------------
class TestIsAvailable:
"""Each plugin's ``is_available()`` reflects env-var presence accurately."""
def test_browserbase_requires_both_api_key_and_project_id(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browserbase")
assert p is not None
assert p.is_available() is False
# API key alone is insufficient.
monkeypatch.setenv("BROWSERBASE_API_KEY", "key")
assert p.is_available() is False
# Both env vars set → available.
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
assert p.is_available() is True
def test_browserbase_project_id_alone_insufficient(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browserbase")
assert p is not None
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
assert p.is_available() is False
def test_browser_use_satisfied_by_api_key(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browser-use")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("BROWSER_USE_API_KEY", "key")
assert p.is_available() is True
def test_firecrawl_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("FIRECRAWL_API_KEY", "key")
assert p.is_available() is True
# ---------------------------------------------------------------------------
# Registry resolution semantics
# ---------------------------------------------------------------------------
class TestRegistryResolution:
"""``_resolve()`` implements the documented three-rule precedence."""
def test_resolve_none_with_no_creds_returns_none(self) -> None:
"""No config, no env → local mode (None)."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
assert _resolve(None) is None
def test_explicit_local_returns_none(self) -> None:
"""``cloud_provider: local`` is a positive choice; short-circuits to None."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
assert _resolve("local") is None
def test_explicit_browserbase_returns_provider_even_when_unavailable(self) -> None:
"""Rule 1: explicit-config wins even when credentials are missing.
This is critical — the dispatcher needs to surface a typed
credentials error rather than silently switching backends.
"""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
provider = _resolve("browserbase")
assert provider is not None
assert provider.name == "browserbase"
assert provider.is_available() is False # confirms "ignoring availability"
def test_explicit_firecrawl_returns_provider_even_when_unavailable(self) -> None:
"""Firecrawl behaves the same as browserbase under explicit config."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
provider = _resolve("firecrawl")
assert provider is not None
assert provider.name == "firecrawl"
def test_explicit_unknown_falls_back_to_auto_detect(self) -> None:
"""Rule 1 miss: unknown name → fall through to legacy walk."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
# With no credentials anywhere, auto-detect should also fail.
assert _resolve("not-a-real-provider") is None
def test_legacy_walk_prefers_browser_use_over_browserbase(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rule 3: walk order is browser-use → browserbase."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
# Both available — browser-use should win.
monkeypatch.setenv("BROWSER_USE_API_KEY", "k1")
monkeypatch.setenv("BROWSERBASE_API_KEY", "k2")
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
provider = _resolve(None)
assert provider is not None
assert provider.name == "browser-use"
def test_legacy_walk_falls_through_to_browserbase(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rule 3: browser-use unavailable → browserbase picked."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
monkeypatch.setenv("BROWSERBASE_API_KEY", "k")
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
provider = _resolve(None)
assert provider is not None
assert provider.name == "browserbase"
def test_firecrawl_not_in_legacy_walk_even_when_only_one_available(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Regression: firecrawl is NEVER auto-selected even when single-eligible.
Pre-PR-#25214, the dispatcher only auto-detected between Browser Use
and Browserbase; firecrawl was reachable solely via explicit
config. We preserve that gate because FIRECRAWL_API_KEY is shared
with the *web* firecrawl plugin — auto-routing a web-extract user
to a paid cloud browser would be a real behaviour regression.
"""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
monkeypatch.setenv("FIRECRAWL_API_KEY", "k")
# Only firecrawl is_available() — but it's not in the legacy walk.
assert _resolve(None) is None
# ---------------------------------------------------------------------------
# Legacy ABC backward-compat aliases (is_configured / provider_name)
# ---------------------------------------------------------------------------
class TestLegacyAbcAliases:
"""is_configured() and provider_name() delegate to the new API."""
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_is_configured_delegates_to_is_available(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider(plugin_name)
assert p is not None
assert p.is_configured() is p.is_available()
@pytest.mark.parametrize(
"plugin_name,expected_label",
[
("browserbase", "Browserbase"),
("browser-use", "Browser Use"),
("firecrawl", "Firecrawl"),
],
)
def test_provider_name_returns_display_name(
self, plugin_name: str, expected_label: str
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider(plugin_name)
assert p is not None
assert p.provider_name() == expected_label
# ---------------------------------------------------------------------------
# Picker integration
# ---------------------------------------------------------------------------
class TestPickerIntegration:
"""`_plugin_browser_providers()` exposes all three plugins as picker rows."""
def test_picker_rows_match_registered_plugins(self) -> None:
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
rows = _plugin_browser_providers()
names = sorted(r.get("browser_provider") for r in rows)
assert names == ["browser-use", "browserbase", "firecrawl"]
def test_picker_rows_carry_post_setup_hook(self) -> None:
"""Every browser plugin row has post_setup='agent_browser' so
selecting it triggers the agent-browser CLI install."""
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
for row in _plugin_browser_providers():
assert row.get("post_setup") == "agent_browser", (
f"plugin row {row['browser_provider']!r} missing post_setup hook"
)
def test_picker_rows_carry_browser_plugin_name_marker(self) -> None:
"""`browser_plugin_name` matches `browser_provider` so downstream
code can route through the registry when it wants to."""
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
for row in _plugin_browser_providers():
assert row.get("browser_plugin_name") == row.get("browser_provider")
tests/plugins/dashboard_auth/test_basic_provider.py
+246
View File
@@ -0,0 +1,246 @@
"""Tests for the BasicAuthProvider plugin (username/password, scrypt, signed
tokens).
Loads the plugin module directly (it's a bundled backend plugin, not on the
import path as a package) and exercises the provider behaviour + the
``register(ctx)`` entry point's config/env resolution and skip reasons.
"""
from __future__ import annotations
import secrets
from unittest.mock import MagicMock
import pytest
import plugins.dashboard_auth.basic as basic_plugin
from hermes_cli.dashboard_auth import (
InvalidCredentialsError,
RefreshExpiredError,
assert_protocol_compliance,
)
@pytest.fixture(scope="module")
def basic():
return basic_plugin
@pytest.fixture(autouse=True)
def _clear_basic_env(monkeypatch):
for var in (
"HERMES_DASHBOARD_BASIC_AUTH_USERNAME",
"HERMES_DASHBOARD_BASIC_AUTH_PASSWORD",
"HERMES_DASHBOARD_BASIC_AUTH_PASSWORD_HASH",
"HERMES_DASHBOARD_BASIC_AUTH_SECRET",
"HERMES_DASHBOARD_BASIC_AUTH_TTL_SECONDS",
):
monkeypatch.delenv(var, raising=False)
# ---------------------------------------------------------------------------
# Hashing
# ---------------------------------------------------------------------------
class TestPasswordHashing:
def test_hash_then_verify_round_trips(self, basic):
h = basic.hash_password("hunter2")
assert h.startswith("scrypt$")
assert basic._verify_password("hunter2", h)
def test_wrong_password_fails(self, basic):
h = basic.hash_password("hunter2")
assert not basic._verify_password("wrong", h)
def test_malformed_hash_returns_false(self, basic):
assert not basic._verify_password("x", "not-a-valid-hash")
assert not basic._verify_password("x", "bcrypt$wrong$scheme")
def test_two_hashes_of_same_password_differ(self, basic):
# Distinct random salts → distinct encoded hashes.
assert basic.hash_password("pw") != basic.hash_password("pw")
# ---------------------------------------------------------------------------
# Provider behaviour
# ---------------------------------------------------------------------------
class TestProvider:
def _make(self, basic, **kw):
h = basic.hash_password("hunter2")
return basic.BasicAuthProvider(
username="admin",
password_hash=h,
secret=secrets.token_bytes(32),
**kw,
)
def test_protocol_compliant(self, basic):
assert assert_protocol_compliance(basic.BasicAuthProvider) is None
def test_supports_password_true(self, basic):
assert basic.BasicAuthProvider.supports_password is True
def test_login_mints_session(self, basic):
p = self._make(basic)
s = p.complete_password_login(username="admin", password="hunter2")
assert s.user_id == "admin"
assert s.provider == "basic"
assert s.access_token and s.refresh_token
def test_bad_credentials_raise(self, basic):
p = self._make(basic)
for u, pw in [("admin", "wrong"), ("ghost", "hunter2"), ("", "")]:
with pytest.raises(InvalidCredentialsError):
p.complete_password_login(username=u, password=pw)
def test_verify_round_trips_and_rejects_tamper(self, basic):
p = self._make(basic)
s = p.complete_password_login(username="admin", password="hunter2")
assert p.verify_session(access_token=s.access_token) is not None
assert p.verify_session(access_token="garbage") is None
def test_access_token_not_accepted_as_refresh(self, basic):
p = self._make(basic)
s = p.complete_password_login(username="admin", password="hunter2")
# A refresh token must not verify as an access token and vice
# versa — the ``kind`` claim is enforced.
assert p.verify_session(access_token=s.refresh_token) is None
with pytest.raises(RefreshExpiredError):
p.refresh_session(refresh_token=s.access_token)
def test_refresh_round_trips(self, basic):
p = self._make(basic)
s = p.complete_password_login(username="admin", password="hunter2")
r = p.refresh_session(refresh_token=s.refresh_token)
assert r.user_id == "admin"
assert p.verify_session(access_token=r.access_token) is not None
def test_refresh_with_garbage_raises(self, basic):
p = self._make(basic)
with pytest.raises(RefreshExpiredError):
p.refresh_session(refresh_token="garbage")
def test_cross_secret_token_does_not_verify(self, basic):
p1 = self._make(basic)
p2 = self._make(basic) # different random secret
s = p1.complete_password_login(username="admin", password="hunter2")
assert p2.verify_session(access_token=s.access_token) is None
def test_revoke_is_silent(self, basic):
p = self._make(basic)
p.revoke_session(refresh_token="anything") # must not raise
def test_oauth_methods_raise_not_implemented(self, basic):
p = self._make(basic)
with pytest.raises(NotImplementedError):
p.start_login(redirect_uri="https://x/auth/callback")
with pytest.raises(NotImplementedError):
p.complete_login(
code="c", state="s", code_verifier="v", redirect_uri="r"
)
def test_construction_validates_inputs(self, basic):
good_hash = basic.hash_password("pw")
with pytest.raises(ValueError):
basic.BasicAuthProvider(
username="", password_hash=good_hash, secret=b"x" * 32
)
with pytest.raises(ValueError):
basic.BasicAuthProvider(
username="admin", password_hash="", secret=b"x" * 32
)
with pytest.raises(ValueError):
basic.BasicAuthProvider(
username="admin", password_hash=good_hash, secret=b"short"
)
# ---------------------------------------------------------------------------
# register() entry point — config/env resolution + skip reasons
# ---------------------------------------------------------------------------
class TestRegister:
def test_skips_when_no_username(self, basic, monkeypatch):
monkeypatch.setattr(basic, "_load_config_basic_auth_section", lambda: {})
ctx = MagicMock()
basic.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
assert "username" in basic.LAST_SKIP_REASON
def test_skips_when_username_but_no_password(self, basic, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_USERNAME", "admin")
monkeypatch.setattr(basic, "_load_config_basic_auth_section", lambda: {})
ctx = MagicMock()
basic.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
assert "password" in basic.LAST_SKIP_REASON
def test_registers_with_env_plaintext_password(self, basic, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_USERNAME", "admin")
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_PASSWORD", "hunter2")
monkeypatch.setattr(basic, "_load_config_basic_auth_section", lambda: {})
ctx = MagicMock()
basic.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
provider = ctx.register_dashboard_auth_provider.call_args.args[0]
assert isinstance(provider, basic.BasicAuthProvider)
# Round-trips: the registered provider authenticates the env creds.
s = provider.complete_password_login(username="admin", password="hunter2")
assert s.user_id == "admin"
assert basic.LAST_SKIP_REASON == ""
def test_registers_with_precomputed_hash(self, basic, monkeypatch):
h = basic.hash_password("s3cret")
monkeypatch.setattr(
basic,
"_load_config_basic_auth_section",
lambda: {"username": "ops", "password_hash": h},
)
ctx = MagicMock()
basic.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
provider = ctx.register_dashboard_auth_provider.call_args.args[0]
assert provider.complete_password_login(
username="ops", password="s3cret"
).user_id == "ops"
def test_env_password_overrides_config(self, basic, monkeypatch):
cfg_hash = basic.hash_password("config-pw")
monkeypatch.setattr(
basic,
"_load_config_basic_auth_section",
lambda: {"username": "admin", "password_hash": cfg_hash},
)
# Env plaintext should win over the config hash.
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_PASSWORD", "env-pw")
ctx = MagicMock()
basic.register(ctx)
provider = ctx.register_dashboard_auth_provider.call_args.args[0]
# env password works ...
assert provider.complete_password_login(
username="admin", password="env-pw"
)
# ... and the config password no longer does.
with pytest.raises(InvalidCredentialsError):
provider.complete_password_login(username="admin", password="config-pw")
def test_explicit_secret_makes_sessions_portable(self, basic, monkeypatch):
# Two providers built from the SAME explicit secret accept each
# other's tokens (the restart-/multi-worker-survival contract).
shared = secrets.token_bytes(32).hex()
monkeypatch.setattr(basic, "_load_config_basic_auth_section", lambda: {})
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_USERNAME", "admin")
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_PASSWORD", "hunter2")
monkeypatch.setenv("HERMES_DASHBOARD_BASIC_AUTH_SECRET", shared)
ctx1, ctx2 = MagicMock(), MagicMock()
basic.register(ctx1)
basic.register(ctx2)
p1 = ctx1.register_dashboard_auth_provider.call_args.args[0]
p2 = ctx2.register_dashboard_auth_provider.call_args.args[0]
s = p1.complete_password_login(username="admin", password="hunter2")
assert p2.verify_session(access_token=s.access_token) is not None
tests/plugins/dashboard_auth/test_nous_provider.py
+848
View File
@@ -0,0 +1,848 @@
"""Tests for the bundled Nous dashboard-auth plugin.
Covers four shapes from Phase 4 of ``.hermes/plans/2026-05-21-dashboard-oauth-auth.md``:
1. Plugin entry-point registration gating (env var checks).
2. ``start_login`` shape (PKCE/state, authorize URL parameters).
3. ``complete_login`` httpx-mocked happy path + error mapping.
4. ``verify_session`` JWT verification — RSA keypair, audience/issuer pinning,
``agent_instance_id`` cross-check, ``oauth_contract_version`` tolerance.
Also exercises ``revoke_session`` (no-op) and ``refresh_session``
(unconditional ``RefreshExpiredError``).
All HTTP is mocked: nothing in this file talks to a real Portal.
"""
from __future__ import annotations
import base64
import hashlib
import json
import time
import urllib.parse
from typing import Any, Dict
from unittest.mock import MagicMock, patch
import httpx
import jwt
import pytest
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
import plugins.dashboard_auth.nous as nous_plugin
from hermes_cli.dashboard_auth import (
InvalidCodeError,
LoginStart,
ProviderError,
RefreshExpiredError,
Session,
assert_protocol_compliance,
)
# ---------------------------------------------------------------------------
# RSA keypair fixture (module-scope — keygen is slow)
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def rsa_keypair() -> Dict[str, Any]:
"""Generate an RS256 keypair + matching JWK for verify_session tests."""
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
private_pem = key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
).decode()
public_numbers = key.public_key().public_numbers()
def _b64url_uint(n: int) -> str:
length = (n.bit_length() + 7) // 8
return (
base64.urlsafe_b64encode(n.to_bytes(length, "big")).rstrip(b"=").decode()
)
jwk = {
"kty": "RSA",
"use": "sig",
"alg": "RS256",
"kid": "test-key-1",
"n": _b64url_uint(public_numbers.n),
"e": _b64url_uint(public_numbers.e),
}
return {"private_pem": private_pem, "jwk": jwk, "kid": jwk["kid"]}
# ---------------------------------------------------------------------------
# Token-mint helper
# ---------------------------------------------------------------------------
def _mint_token(
rsa_keypair: Dict[str, Any],
*,
iss: str = "https://portal.example.com",
aud: str = "agent:inst123",
sub: str = "usr_abc",
agent_instance_id: str | None = "inst123",
oauth_contract_version: Any = 1,
org_id: str | None = "org_xyz",
scope: str = "agent_dashboard:access",
ttl_seconds: int = 900,
extra_claims: Dict[str, Any] | None = None,
) -> str:
now = int(time.time())
claims = {
"iss": iss,
"aud": aud,
"sub": sub,
"iat": now,
"exp": now + ttl_seconds,
"scope": scope,
}
if agent_instance_id is not None:
claims["agent_instance_id"] = agent_instance_id
if oauth_contract_version is not None:
claims["oauth_contract_version"] = oauth_contract_version
if org_id is not None:
claims["org_id"] = org_id
if extra_claims:
claims.update(extra_claims)
return jwt.encode(
claims,
rsa_keypair["private_pem"],
algorithm="RS256",
headers={"kid": rsa_keypair["kid"]},
)
def _patched_jwks(provider: nous_plugin.NousDashboardAuthProvider, rsa_keypair):
"""Patch the provider's JWKS client to return our fixture key."""
fake_key = MagicMock()
fake_key.key = serialization.load_pem_private_key(
rsa_keypair["private_pem"].encode(), password=None
).public_key()
fake_client = MagicMock()
fake_client.get_signing_key_from_jwt.return_value = fake_key
provider._jwks_client = fake_client
# ---------------------------------------------------------------------------
# Provider construction
# ---------------------------------------------------------------------------
class TestConstruction:
def test_protocol_compliance(self):
assert_protocol_compliance(nous_plugin.NousDashboardAuthProvider)
def test_name_and_display(self):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst1", portal_url="https://portal.example.com"
)
assert p.name == "nous"
assert p.display_name == "Nous Research"
def test_extracts_agent_instance_id(self):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:abc-123", portal_url="https://portal.example.com"
)
assert p._agent_instance_id == "abc-123"
def test_strips_trailing_slash_from_portal_url(self):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:x", portal_url="https://portal.example.com/"
)
assert p._portal_url == "https://portal.example.com"
def test_rejects_malformed_client_id(self):
with pytest.raises(ValueError, match="agent:"):
nous_plugin.NousDashboardAuthProvider(
client_id="hermes-dashboard", portal_url="https://x"
)
# ---------------------------------------------------------------------------
# Plugin entry point: env-gated registration
# ---------------------------------------------------------------------------
class TestPluginRegister:
def test_skips_when_client_id_missing(self, monkeypatch):
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
# Skip reason is surfaced for the gate's fail-closed message.
assert "HERMES_DASHBOARD_OAUTH_CLIENT_ID" in nous_plugin.LAST_SKIP_REASON
def test_registers_with_default_portal_url_when_only_client_id_set(
self, monkeypatch
):
"""Phase 7 follow-up: HERMES_DASHBOARD_PORTAL_URL is optional —
defaults to the production Nous Portal. The user shouldn't have
to set it for the common production deployment path."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert isinstance(registered, nous_plugin.NousDashboardAuthProvider)
assert registered._portal_url == "https://portal.nousresearch.com"
# Skip reason cleared on successful registration.
assert nous_plugin.LAST_SKIP_REASON == ""
def test_skips_when_client_id_malformed(self, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "hermes-dashboard")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "https://p.example")
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
# Skip reason names the offending value + contract shape.
assert "agent:" in nous_plugin.LAST_SKIP_REASON
assert "hermes-dashboard" in nous_plugin.LAST_SKIP_REASON
def test_registers_with_explicit_portal_url(self, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "https://p.example")
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:inst1"
assert registered._portal_url == "https://p.example"
def test_strips_whitespace_from_env_vars(self, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", " agent:x ")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", " https://p.example ")
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
def test_empty_portal_url_env_uses_default(self, monkeypatch):
"""Explicit empty string still falls back to the production
default — same handling as 'unset' so an empty Fly secret can't
accidentally point the dashboard at nowhere."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "")
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._portal_url == "https://portal.nousresearch.com"
# ---------------------------------------------------------------------------
# Plugin entry point: config.yaml + env-override precedence
# ---------------------------------------------------------------------------
class TestConfigYamlSource:
"""``dashboard.oauth.{client_id,portal_url}`` in ``config.yaml`` is the
canonical surface for these settings. ``HERMES_DASHBOARD_OAUTH_CLIENT_ID``
and ``HERMES_DASHBOARD_PORTAL_URL`` are operator overrides that win when
set — this is the contract Fly.io's platform-secret injection relies on,
and the contract that lets local devs experiment without setting env
vars.
Each test pins exactly one tier of the precedence chain so a regression
that flips the order is caught:
env (when truthy) > config.yaml (when truthy) > plugin default
"""
@pytest.fixture
def patch_config(self, monkeypatch):
"""Yield a callable that replaces ``hermes_cli.config.load_config``
with a stub returning the given dict. Tests pass the intended
``dashboard.oauth`` block; the stub returns the wrapping structure."""
def _set(oauth_block: Dict[str, Any] | None) -> None:
cfg = {}
if oauth_block is not None:
cfg = {"dashboard": {"oauth": oauth_block}}
monkeypatch.setattr(
"hermes_cli.config.load_config", lambda: cfg
)
return _set
def test_config_yaml_only_client_id_registers(self, patch_config, monkeypatch):
"""No env var, only config.yaml — plugin reads from config and
registers successfully. This is the path Teknium's review pushed
for (".env is for secrets only")."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
patch_config({"client_id": "agent:from-config"})
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-config"
# Defaults to production portal URL when neither config nor env
# specifies one.
assert registered._portal_url == "https://portal.nousresearch.com"
def test_config_yaml_client_id_and_portal_url(self, patch_config, monkeypatch):
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
patch_config({
"client_id": "agent:from-config",
"portal_url": "https://staging.portal.example",
})
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-config"
assert registered._portal_url == "https://staging.portal.example"
def test_env_overrides_config_client_id(self, patch_config, monkeypatch):
"""Env wins. Critical for Fly.io: the Portal injects
HERMES_DASHBOARD_OAUTH_CLIENT_ID at deploy time and we MUST
honour it even if a stale config.yaml ships in the image."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:from-env")
patch_config({"client_id": "agent:from-config"})
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-env", (
"env var must override config.yaml — Fly secret injection "
"depends on this precedence"
)
def test_env_overrides_config_portal_url(self, patch_config, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:x")
monkeypatch.setenv(
"HERMES_DASHBOARD_PORTAL_URL", "https://env.portal.example",
)
patch_config({
"client_id": "agent:x",
"portal_url": "https://config.portal.example",
})
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._portal_url == "https://env.portal.example"
def test_empty_env_string_does_not_shadow_config(
self, patch_config, monkeypatch
):
"""``HERMES_DASHBOARD_OAUTH_CLIENT_ID=`` (set but empty) is
common in CI/Fly when a secret is provisioned-but-not-populated.
It MUST NOT shadow a valid config.yaml value with an empty
string — operators would lose the gate."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "")
patch_config({"client_id": "agent:from-config"})
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-config"
def test_neither_source_skips_with_helpful_reason(
self, patch_config, monkeypatch
):
"""Neither env nor config.yaml set — skip with a reason that
mentions BOTH surfaces so operators don't guess wrong about
which one to populate."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
patch_config(None)
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
# Old behaviour: skip reason mentions the env var.
assert "HERMES_DASHBOARD_OAUTH_CLIENT_ID" in nous_plugin.LAST_SKIP_REASON
# New behaviour: skip reason ALSO mentions the config.yaml path
# so the user knows it's a valid alternative.
assert "dashboard.oauth.client_id" in nous_plugin.LAST_SKIP_REASON, (
f"skip reason omits the config.yaml surface — operators "
f"won't know it exists. got: {nous_plugin.LAST_SKIP_REASON!r}"
)
def test_config_yaml_load_failure_falls_through_cleanly(
self, monkeypatch
):
"""If load_config() raises (e.g. malformed YAML, IOError), the
plugin must not crash — it falls through to the env-only path
and either succeeds (if env is set) or surfaces the standard
'not set' skip reason."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
def _broken_load():
raise OSError("config.yaml not readable")
monkeypatch.setattr(
"hermes_cli.config.load_config", _broken_load
)
ctx = MagicMock()
# Must not raise.
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
def test_config_yaml_with_non_dict_oauth_section(
self, monkeypatch
):
"""cfg_get handles 'config has a string where a section was
expected' robustly. Verify the plugin inherits that resilience
so a malformed user config doesn't crash startup."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"dashboard": {"oauth": "wrong type"}},
)
ctx = MagicMock()
nous_plugin.register(ctx)
# Falls through to the no-env-and-no-config path.
ctx.register_dashboard_auth_provider.assert_not_called()
# ---------------------------------------------------------------------------
# start_login
# ---------------------------------------------------------------------------
class TestStartLogin:
@pytest.fixture
def provider(self):
return nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst1", portal_url="https://portal.example.com"
)
def test_returns_login_start(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert isinstance(result, LoginStart)
def test_redirect_url_targets_portal_authorize(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert result.redirect_url.startswith(
"https://portal.example.com/oauth/authorize?"
)
def test_authorize_url_has_required_params(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
assert params["response_type"] == "code"
assert params["client_id"] == "agent:inst1"
assert params["redirect_uri"] == "https://hermes.fly.dev/auth/callback"
assert params["scope"] == "agent_dashboard:access"
assert params["code_challenge_method"] == "S256"
assert "state" in params
assert "code_challenge" in params
def test_code_verifier_in_cookie_payload_43_to_128_chars(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert "hermes_session_pkce" in result.cookie_payload
pkce = result.cookie_payload["hermes_session_pkce"]
# Shape: ``state=…;verifier=…`` (matches stub-provider convention so
# the auth-route layer's parser works uniformly across providers).
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
verifier = parts["verifier"]
# RFC 7636 §4.1
assert 43 <= len(verifier) <= 128
def test_state_in_cookie_payload_matches_url_param(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
assert parts["state"] == params["state"]
def test_code_challenge_is_s256_of_verifier(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
verifier = parts["verifier"]
expected_challenge = (
base64.urlsafe_b64encode(
hashlib.sha256(verifier.encode("ascii")).digest()
)
.rstrip(b"=")
.decode()
)
assert params["code_challenge"] == expected_challenge
def test_two_calls_produce_different_state_and_verifier(self, provider):
a = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
b = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert a.cookie_payload["hermes_session_pkce"] != b.cookie_payload[
"hermes_session_pkce"
]
def test_rejects_non_http_scheme(self, provider):
with pytest.raises(ProviderError, match="http"):
provider.start_login(redirect_uri="ftp://x/auth/callback")
def test_allows_http_with_arbitrary_host(self, provider):
# http:// is permitted for any host now, not just localhost — the
# Portal-side check is authoritative on which redirect_uris are
# accepted; this client-side fast-fail must not reject self-hosted
# dashboards reached over plain HTTP (LAN IPs, internal hostnames,
# TLS-terminating reverse proxies). Should not raise.
provider.start_login(redirect_uri="http://hermes.fly.dev/auth/callback")
provider.start_login(redirect_uri="http://192.168.1.50:8080/auth/callback")
provider.start_login(redirect_uri="http://my-internal-host/auth/callback")
def test_allows_http_localhost(self, provider):
# Should not raise.
provider.start_login(redirect_uri="http://localhost:8080/auth/callback")
provider.start_login(redirect_uri="http://127.0.0.1:8080/auth/callback")
def test_rejects_wrong_callback_path(self, provider):
with pytest.raises(ProviderError, match="/auth/callback"):
provider.start_login(redirect_uri="https://x.example/oauth/cb")
# ---------------------------------------------------------------------------
# complete_login (httpx mocked)
# ---------------------------------------------------------------------------
class TestCompleteLogin:
@pytest.fixture
def provider(self, rsa_keypair):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst123", portal_url="https://portal.example.com"
)
_patched_jwks(p, rsa_keypair)
return p
def _mock_post(self, status_code: int, body: Any, *, ctype: str = "application/json"):
resp = MagicMock(spec=httpx.Response)
resp.status_code = status_code
if isinstance(body, dict):
resp.text = json.dumps(body)
resp.json = MagicMock(return_value=body)
else:
resp.text = body
# _parse_json_body bails on non-application/json before .json()
# is called, but be safe for callers that pass a non-dict body
# with ctype=application/json.
resp.json = MagicMock(side_effect=ValueError("not json"))
resp.headers = {"content-type": ctype}
return resp
def test_happy_path_returns_session(self, provider, rsa_keypair):
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200,
{
"access_token": access_token,
"token_type": "Bearer",
"refresh_token": "rt_initial_value",
},
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
session = provider.complete_login(
code="abc",
state="state-val",
code_verifier="vfy",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
assert isinstance(session, Session)
assert session.user_id == "usr_abc"
assert session.provider == "nous"
assert session.access_token == access_token
# The dashboard auth-code grant now issues a refresh token (NAS #293);
# complete_login must surface it so the middleware persists it.
assert session.refresh_token == "rt_initial_value"
assert session.org_id == "org_xyz"
assert session.email == ""
assert session.display_name == ""
def test_happy_path_tolerates_missing_refresh_token(self, provider, rsa_keypair):
# If Portal omits refresh_token (older deploy), the session is still
# valid as access-token-only; refresh_token defaults to "".
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200, {"access_token": access_token, "token_type": "Bearer"}
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
session = provider.complete_login(
code="abc",
state="state-val",
code_verifier="vfy",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
assert session.refresh_token == ""
def test_400_raises_invalid_code(self, provider):
mock_resp = self._mock_post(400, {"error": "invalid_grant"})
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(InvalidCodeError, match="invalid_grant"):
provider.complete_login(
code="bad", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_500_raises_provider_error(self, provider):
mock_resp = self._mock_post(500, "internal server error", ctype="text/plain")
mock_resp.text = "internal server error"
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError, match="500"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_missing_access_token_raises(self, provider):
mock_resp = self._mock_post(200, {"token_type": "Bearer"})
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError, match="access_token"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_unexpected_token_type_raises(self, provider, rsa_keypair):
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200, {"access_token": access_token, "token_type": "DPoP"}
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError, match="token_type"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_network_error_raises_provider_error(self, provider):
with patch(
"plugins.dashboard_auth.nous.httpx.post",
side_effect=httpx.ConnectError("conn refused"),
):
with pytest.raises(ProviderError, match="unreachable"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_captures_refresh_token_if_present_forward_compat(
self, provider, rsa_keypair
):
"""Forward-compat: contract V1 doesn't issue, but if a future Portal
does, we should preserve it in the Session for later use."""
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200,
{
"access_token": access_token,
"token_type": "Bearer",
"refresh_token": "rt-opaque",
},
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
session = provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
assert session.refresh_token == "rt-opaque"
# ---------------------------------------------------------------------------
# verify_session
# ---------------------------------------------------------------------------
class TestVerifySession:
@pytest.fixture
def provider(self, rsa_keypair):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst123", portal_url="https://portal.example.com"
)
_patched_jwks(p, rsa_keypair)
return p
def test_happy_path_returns_session(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair)
session = provider.verify_session(access_token=token)
assert session is not None
assert session.user_id == "usr_abc"
assert session.org_id == "org_xyz"
def test_expired_token_returns_none(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, ttl_seconds=-1)
assert provider.verify_session(access_token=token) is None
def test_wrong_audience_raises_provider_error(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, aud="agent:other-instance")
with pytest.raises(ProviderError, match="verification failed"):
provider.verify_session(access_token=token)
def test_wrong_issuer_raises_provider_error(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, iss="https://evil.example")
with pytest.raises(ProviderError, match="verification failed"):
provider.verify_session(access_token=token)
def test_verification_failure_message_surfaces_token_claims(
self, provider, rsa_keypair
):
"""Operators need to see the actual iss/aud the token carries to debug
config drift between HERMES_DASHBOARD_PORTAL_URL/CLIENT_ID and Portal."""
token = _mint_token(rsa_keypair, iss="https://evil.example")
with pytest.raises(ProviderError) as excinfo:
provider.verify_session(access_token=token)
msg = str(excinfo.value)
# Both the observed (token) and expected (configured) values appear.
assert "'https://evil.example'" in msg
assert "'https://portal.example.com'" in msg # configured portal URL
def test_missing_sub_raises(self, provider, rsa_keypair):
# PyJWT's "require" set includes sub, so this surfaces as
# InvalidTokenError → ProviderError before we ever touch _session_from_claims.
token = _mint_token(rsa_keypair, sub="")
# Empty sub still encodes successfully; PyJWT's require check only
# asserts presence. Our own _session_from_claims rejects empty.
with pytest.raises(ProviderError, match="sub"):
provider.verify_session(access_token=token)
def test_agent_instance_id_mismatch_rejected(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, agent_instance_id="some-other-id")
with pytest.raises(ProviderError, match="agent_instance_id mismatch"):
provider.verify_session(access_token=token)
def test_agent_instance_id_missing_is_tolerated(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, agent_instance_id=None)
session = provider.verify_session(access_token=token)
assert session is not None
def test_contract_version_missing_warns_but_succeeds(
self, provider, rsa_keypair, caplog
):
import logging
token = _mint_token(rsa_keypair, oauth_contract_version=None)
with caplog.at_level(logging.WARNING, logger="plugins.dashboard_auth.nous"):
session = provider.verify_session(access_token=token)
assert session is not None
assert any(
"oauth_contract_version" in r.message for r in caplog.records
)
def test_contract_version_mismatch_rejected(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, oauth_contract_version=2)
with pytest.raises(ProviderError, match="oauth_contract_version"):
provider.verify_session(access_token=token)
def test_jwks_unreachable_raises_provider_error(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair)
# Replace the patched client so it raises.
bad_client = MagicMock()
bad_client.get_signing_key_from_jwt.side_effect = jwt.PyJWKClientError(
"fetch failed"
)
provider._jwks_client = bad_client
with pytest.raises(ProviderError, match="JWKS"):
provider.verify_session(access_token=token)
# ---------------------------------------------------------------------------
# refresh_session + revoke_session
# ---------------------------------------------------------------------------
class TestRefreshAndRevoke:
@pytest.fixture
def provider(self, rsa_keypair):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst123", portal_url="https://portal.example.com"
)
_patched_jwks(p, rsa_keypair)
return p
def _mock_post(self, status_code, body, *, ctype="application/json"):
resp = MagicMock(spec=httpx.Response)
resp.status_code = status_code
if isinstance(body, dict):
resp.text = json.dumps(body)
resp.json = MagicMock(return_value=body)
else:
resp.text = body
resp.json = MagicMock(side_effect=ValueError("not json"))
resp.headers = {"content-type": ctype}
return resp
def test_refresh_happy_path_returns_rotated_session(self, provider, rsa_keypair):
# Portal returns a fresh access token AND a rotated refresh token.
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200,
{
"access_token": access_token,
"token_type": "Bearer",
"refresh_token": "rt_rotated_value",
},
)
with patch(
"plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp
) as mock_post:
session = provider.refresh_session(refresh_token="rt_old_value")
assert isinstance(session, Session)
assert session.access_token == access_token
# The ROTATED refresh token must be surfaced so the middleware can
# persist it back to the cookie.
assert session.refresh_token == "rt_rotated_value"
assert session.provider == "nous"
# Posts grant_type=refresh_token with the RT in BOTH the body (Portal's
# schema requires it there) and the X-Refresh-Token header (log
# redaction). Verified against the live preview deploy.
_, kwargs = mock_post.call_args
assert kwargs["data"]["grant_type"] == "refresh_token"
assert kwargs["data"]["client_id"] == "agent:inst123"
assert kwargs["data"]["refresh_token"] == "rt_old_value"
assert kwargs["headers"]["x-nous-refresh-token"] == "rt_old_value"
def test_refresh_400_raises_refresh_expired(self, provider):
# Expired / revoked / reuse-detected RT → Portal 400 → force re-login.
mock_resp = self._mock_post(400, {"error": "invalid_grant"})
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(RefreshExpiredError, match="invalid_grant"):
provider.refresh_session(refresh_token="rt_dead")
def test_refresh_empty_token_raises_refresh_expired_without_network(self, provider):
# No RT present — fail fast as a dead session, never hit the network.
with patch("plugins.dashboard_auth.nous.httpx.post") as mock_post:
with pytest.raises(RefreshExpiredError):
provider.refresh_session(refresh_token="")
mock_post.assert_not_called()
def test_refresh_network_error_raises_provider_error(self, provider):
with patch(
"plugins.dashboard_auth.nous.httpx.post",
side_effect=httpx.RequestError("boom"),
):
with pytest.raises(ProviderError, match="unreachable"):
provider.refresh_session(refresh_token="rt_x")
def test_refresh_500_raises_provider_error(self, provider):
mock_resp = self._mock_post(500, "oops", ctype="text/plain")
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError):
provider.refresh_session(refresh_token="rt_x")
def test_revoke_is_noop(self, provider):
# Must not raise; returns None implicitly.
assert provider.revoke_session(refresh_token="anything") is None
assert provider.revoke_session(refresh_token="") is None
tests/plugins/dashboard_auth/test_self_hosted_provider.py
+882
View File
@@ -0,0 +1,882 @@
"""Tests for the bundled self-hosted OIDC dashboard-auth plugin.
Covers, by analogy with ``test_nous_provider.py``:
1. Plugin entry-point registration gating (env + config.yaml precedence).
2. ``start_login`` shape (PKCE/state, authorize URL parameters, OIDC discovery).
3. ``complete_login`` httpx-mocked happy path + error mapping (ID-token grant).
4. ``verify_session`` ID-token verification — RSA keypair, audience/issuer
pinning, standard OIDC claim mapping (sub/email/name/groups).
5. ``refresh_session`` rotation + error mapping, ``revoke_session`` (RFC 7009).
6. OIDC discovery: endpoint extraction, issuer pinning, https enforcement.
All HTTP is mocked: nothing here talks to a real IDP.
"""
from __future__ import annotations
import base64
import hashlib
import json
import time
import urllib.parse
from typing import Any, Dict
from unittest.mock import MagicMock, patch
import httpx
import jwt
import pytest
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
import plugins.dashboard_auth.self_hosted as oidc_plugin
from hermes_cli.dashboard_auth import (
InvalidCodeError,
LoginStart,
ProviderError,
RefreshExpiredError,
Session,
assert_protocol_compliance,
)
_ISSUER = "https://auth.example.com/application/o/hermes"
_CLIENT_ID = "hermes-dashboard"
_DISCOVERY_DOC = {
"issuer": _ISSUER,
"authorization_endpoint": f"{_ISSUER}/authorize",
"token_endpoint": f"{_ISSUER}/token",
"jwks_uri": f"{_ISSUER}/jwks",
"revocation_endpoint": f"{_ISSUER}/revoke",
}
# ---------------------------------------------------------------------------
# RSA keypair fixture (module-scope — keygen is slow)
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def rsa_keypair() -> Dict[str, Any]:
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
private_pem = key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
).decode()
public_numbers = key.public_key().public_numbers()
def _b64url_uint(n: int) -> str:
length = (n.bit_length() + 7) // 8
return (
base64.urlsafe_b64encode(n.to_bytes(length, "big")).rstrip(b"=").decode()
)
jwk = {
"kty": "RSA",
"use": "sig",
"alg": "RS256",
"kid": "test-key-1",
"n": _b64url_uint(public_numbers.n),
"e": _b64url_uint(public_numbers.e),
}
return {"private_pem": private_pem, "jwk": jwk, "kid": jwk["kid"]}
# ---------------------------------------------------------------------------
# Token-mint helper — standard OIDC ID-token claims
# ---------------------------------------------------------------------------
def _mint_id_token(
rsa_keypair: Dict[str, Any],
*,
iss: str = _ISSUER,
aud: str = _CLIENT_ID,
sub: str = "usr_abc",
email: str | None = "alice@example.com",
name: str | None = "Alice Example",
groups: Any = None,
org_id: str | None = None,
ttl_seconds: int = 900,
extra_claims: Dict[str, Any] | None = None,
) -> str:
now = int(time.time())
claims: Dict[str, Any] = {
"iss": iss,
"aud": aud,
"sub": sub,
"iat": now,
"exp": now + ttl_seconds,
}
if email is not None:
claims["email"] = email
if name is not None:
claims["name"] = name
if groups is not None:
claims["groups"] = groups
if org_id is not None:
claims["org_id"] = org_id
if extra_claims:
claims.update(extra_claims)
return jwt.encode(
claims,
rsa_keypair["private_pem"],
algorithm="RS256",
headers={"kid": rsa_keypair["kid"]},
)
def _make_provider(rsa_keypair, *, scopes: str | None = None):
"""Construct a provider with discovery + JWKS stubbed (no network)."""
kwargs: Dict[str, Any] = {"issuer": _ISSUER, "client_id": _CLIENT_ID}
if scopes is not None:
kwargs["scopes"] = scopes
p = oidc_plugin.SelfHostedOIDCProvider(**kwargs)
# Pre-seed discovery so nothing hits the network.
p._discovery = dict(_DISCOVERY_DOC)
p._discovery_fetched_at = time.time()
# Patch the JWKS client to return our fixture key.
fake_key = MagicMock()
fake_key.key = serialization.load_pem_private_key(
rsa_keypair["private_pem"].encode(), password=None
).public_key()
fake_client = MagicMock()
fake_client.get_signing_key_from_jwt.return_value = fake_key
p._jwks_client = fake_client
return p
def _mock_post(status_code: int, body: Any, *, ctype: str = "application/json"):
resp = MagicMock(spec=httpx.Response)
resp.status_code = status_code
if isinstance(body, dict):
resp.text = json.dumps(body)
resp.json = MagicMock(return_value=body)
else:
resp.text = body
resp.json = MagicMock(side_effect=ValueError("not json"))
resp.headers = {"content-type": ctype}
return resp
# ---------------------------------------------------------------------------
# Construction
# ---------------------------------------------------------------------------
class TestConstruction:
def test_protocol_compliance(self):
assert_protocol_compliance(oidc_plugin.SelfHostedOIDCProvider)
def test_name_and_display(self):
p = oidc_plugin.SelfHostedOIDCProvider(issuer=_ISSUER, client_id=_CLIENT_ID)
assert p.name == "self-hosted"
assert p.display_name == "Self-Hosted OIDC"
def test_strips_trailing_slash_from_issuer(self):
p = oidc_plugin.SelfHostedOIDCProvider(
issuer=_ISSUER + "/", client_id=_CLIENT_ID
)
assert p._issuer == _ISSUER
def test_requires_issuer(self):
with pytest.raises(ValueError, match="issuer"):
oidc_plugin.SelfHostedOIDCProvider(issuer="", client_id=_CLIENT_ID)
def test_requires_client_id(self):
with pytest.raises(ValueError, match="client_id"):
oidc_plugin.SelfHostedOIDCProvider(issuer=_ISSUER, client_id="")
def test_rejects_non_https_issuer(self):
with pytest.raises(ProviderError, match="https"):
oidc_plugin.SelfHostedOIDCProvider(
issuer="http://auth.example.com", client_id=_CLIENT_ID
)
def test_allows_http_localhost_issuer(self):
# Local dev against a loopback IDP is allowed.
p = oidc_plugin.SelfHostedOIDCProvider(
issuer="http://localhost:9000", client_id=_CLIENT_ID
)
assert p._issuer == "http://localhost:9000"
def test_default_scopes(self):
p = oidc_plugin.SelfHostedOIDCProvider(issuer=_ISSUER, client_id=_CLIENT_ID)
assert p._scopes == "openid profile email"
def test_empty_scopes_falls_back_to_default(self):
p = oidc_plugin.SelfHostedOIDCProvider(
issuer=_ISSUER, client_id=_CLIENT_ID, scopes=" "
)
assert p._scopes == "openid profile email"
# ---------------------------------------------------------------------------
# OIDC discovery
# ---------------------------------------------------------------------------
class TestDiscovery:
def _provider(self):
return oidc_plugin.SelfHostedOIDCProvider(
issuer=_ISSUER, client_id=_CLIENT_ID
)
def _mock_get(self, status_code, body, *, ctype="application/json"):
resp = MagicMock(spec=httpx.Response)
resp.status_code = status_code
resp.json = MagicMock(return_value=body)
resp.text = json.dumps(body) if isinstance(body, dict) else str(body)
resp.headers = {"content-type": ctype}
return resp
def test_discovery_url(self):
p = self._provider()
assert p._discovery_url() == (
f"{_ISSUER}/.well-known/openid-configuration"
)
def test_fetches_and_caches(self):
p = self._provider()
mock_resp = self._mock_get(200, dict(_DISCOVERY_DOC))
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get", return_value=mock_resp
) as mock_get:
disco1 = p._get_discovery()
disco2 = p._get_discovery()
assert disco1["token_endpoint"] == f"{_ISSUER}/token"
assert disco1["authorization_endpoint"] == f"{_ISSUER}/authorize"
assert disco1["jwks_uri"] == f"{_ISSUER}/jwks"
assert disco1["revocation_endpoint"] == f"{_ISSUER}/revoke"
# Cached — only one network call.
assert mock_get.call_count == 1
assert disco2 is disco1
def test_discovery_404_raises(self):
p = self._provider()
mock_resp = self._mock_get(404, {})
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get", return_value=mock_resp
):
with pytest.raises(ProviderError, match="404"):
p._get_discovery()
def test_discovery_unreachable_raises(self):
p = self._provider()
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get",
side_effect=httpx.ConnectError("no route"),
):
with pytest.raises(ProviderError, match="unreachable"):
p._get_discovery()
def test_discovery_missing_endpoint_raises(self):
p = self._provider()
doc = dict(_DISCOVERY_DOC)
del doc["token_endpoint"]
mock_resp = self._mock_get(200, doc)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get", return_value=mock_resp
):
with pytest.raises(ProviderError, match="token_endpoint"):
p._get_discovery()
def test_discovery_issuer_mismatch_raises(self):
p = self._provider()
doc = dict(_DISCOVERY_DOC)
doc["issuer"] = "https://evil.example"
mock_resp = self._mock_get(200, doc)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get", return_value=mock_resp
):
with pytest.raises(ProviderError, match="issuer mismatch"):
p._get_discovery()
def test_discovery_issuer_trailing_slash_tolerated(self):
p = self._provider()
doc = dict(_DISCOVERY_DOC)
doc["issuer"] = _ISSUER + "/" # only a trailing-slash difference
mock_resp = self._mock_get(200, doc)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get", return_value=mock_resp
):
disco = p._get_discovery()
assert disco["token_endpoint"] == f"{_ISSUER}/token"
def test_discovery_rejects_non_https_endpoint(self):
p = self._provider()
doc = dict(_DISCOVERY_DOC)
doc["token_endpoint"] = "http://auth.example.com/token" # not loopback
mock_resp = self._mock_get(200, doc)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.get", return_value=mock_resp
):
with pytest.raises(ProviderError, match="https"):
p._get_discovery()
# ---------------------------------------------------------------------------
# start_login
# ---------------------------------------------------------------------------
class TestStartLogin:
@pytest.fixture
def provider(self, rsa_keypair):
return _make_provider(rsa_keypair)
def test_returns_login_start(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
assert isinstance(result, LoginStart)
def test_redirect_url_targets_authorize_endpoint(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
assert result.redirect_url.startswith(f"{_ISSUER}/authorize?")
def test_authorize_url_has_required_params(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
assert params["response_type"] == "code"
assert params["client_id"] == _CLIENT_ID
assert params["redirect_uri"] == "https://hermes.example/auth/callback"
assert params["scope"] == "openid profile email"
assert params["code_challenge_method"] == "S256"
assert "state" in params
assert "code_challenge" in params
def test_custom_scopes_used(self, rsa_keypair):
provider = _make_provider(rsa_keypair, scopes="openid email groups")
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
assert params["scope"] == "openid email groups"
def test_code_verifier_length(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
assert 43 <= len(parts["verifier"]) <= 128 # RFC 7636 §4.1
def test_state_in_cookie_matches_url(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
assert parts["state"] == params["state"]
def test_code_challenge_is_s256_of_verifier(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.example/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
expected = (
base64.urlsafe_b64encode(
hashlib.sha256(parts["verifier"].encode("ascii")).digest()
)
.rstrip(b"=")
.decode()
)
assert params["code_challenge"] == expected
def test_two_calls_differ(self, provider):
a = provider.start_login(redirect_uri="https://hermes.example/auth/callback")
b = provider.start_login(redirect_uri="https://hermes.example/auth/callback")
assert (
a.cookie_payload["hermes_session_pkce"]
!= b.cookie_payload["hermes_session_pkce"]
)
def test_rejects_wrong_callback_path(self, provider):
with pytest.raises(ProviderError, match="/auth/callback"):
provider.start_login(redirect_uri="https://x.example/oauth/cb")
def test_allows_http_localhost_redirect(self, provider):
provider.start_login(redirect_uri="http://localhost:8080/auth/callback")
provider.start_login(redirect_uri="http://127.0.0.1:8080/auth/callback")
# ---------------------------------------------------------------------------
# complete_login
# ---------------------------------------------------------------------------
class TestCompleteLogin:
@pytest.fixture
def provider(self, rsa_keypair):
return _make_provider(rsa_keypair)
def test_happy_path_returns_session(self, provider, rsa_keypair):
id_token = _mint_id_token(rsa_keypair)
mock_resp = _mock_post(
200,
{
"access_token": "opaque-at",
"id_token": id_token,
"token_type": "Bearer",
"refresh_token": "rt_initial",
},
)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
session = provider.complete_login(
code="abc",
state="s",
code_verifier="vfy",
redirect_uri="https://hermes.example/auth/callback",
)
assert isinstance(session, Session)
assert session.user_id == "usr_abc"
assert session.provider == "self-hosted"
assert session.email == "alice@example.com"
assert session.display_name == "Alice Example"
# The verified ID token is stored in the access_token slot.
assert session.access_token == id_token
assert session.refresh_token == "rt_initial"
def test_tolerates_missing_refresh_token(self, provider, rsa_keypair):
id_token = _mint_id_token(rsa_keypair)
mock_resp = _mock_post(
200, {"id_token": id_token, "token_type": "Bearer"}
)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
session = provider.complete_login(
code="abc",
state="s",
code_verifier="vfy",
redirect_uri="https://hermes.example/auth/callback",
)
assert session.refresh_token == ""
def test_missing_id_token_raises(self, provider):
mock_resp = _mock_post(
200, {"access_token": "opaque", "token_type": "Bearer"}
)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
with pytest.raises(ProviderError, match="id_token"):
provider.complete_login(
code="x",
state="s",
code_verifier="v",
redirect_uri="https://hermes.example/auth/callback",
)
def test_400_raises_invalid_code(self, provider):
mock_resp = _mock_post(400, {"error": "invalid_grant"})
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
with pytest.raises(InvalidCodeError, match="invalid_grant"):
provider.complete_login(
code="bad",
state="s",
code_verifier="v",
redirect_uri="https://hermes.example/auth/callback",
)
def test_500_raises_provider_error(self, provider):
mock_resp = _mock_post(500, "boom", ctype="text/plain")
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
with pytest.raises(ProviderError, match="500"):
provider.complete_login(
code="x",
state="s",
code_verifier="v",
redirect_uri="https://hermes.example/auth/callback",
)
def test_network_error_raises_provider_error(self, provider):
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post",
side_effect=httpx.ConnectError("conn refused"),
):
with pytest.raises(ProviderError, match="unreachable"):
provider.complete_login(
code="x",
state="s",
code_verifier="v",
redirect_uri="https://hermes.example/auth/callback",
)
def test_unexpected_token_type_raises(self, provider, rsa_keypair):
id_token = _mint_id_token(rsa_keypair)
mock_resp = _mock_post(
200, {"id_token": id_token, "token_type": "DPoP"}
)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
with pytest.raises(ProviderError, match="token_type"):
provider.complete_login(
code="x",
state="s",
code_verifier="v",
redirect_uri="https://hermes.example/auth/callback",
)
def test_posts_authorization_code_grant(self, provider, rsa_keypair):
id_token = _mint_id_token(rsa_keypair)
mock_resp = _mock_post(200, {"id_token": id_token, "token_type": "Bearer"})
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
) as mock_post:
provider.complete_login(
code="the-code",
state="s",
code_verifier="the-verifier",
redirect_uri="https://hermes.example/auth/callback",
)
_, kwargs = mock_post.call_args
assert kwargs["data"]["grant_type"] == "authorization_code"
assert kwargs["data"]["code"] == "the-code"
assert kwargs["data"]["code_verifier"] == "the-verifier"
assert kwargs["data"]["client_id"] == _CLIENT_ID
# ---------------------------------------------------------------------------
# verify_session
# ---------------------------------------------------------------------------
class TestVerifySession:
@pytest.fixture
def provider(self, rsa_keypair):
return _make_provider(rsa_keypair)
def test_happy_path(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair)
session = provider.verify_session(access_token=token)
assert session is not None
assert session.user_id == "usr_abc"
assert session.email == "alice@example.com"
assert session.display_name == "Alice Example"
def test_expired_returns_none(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, ttl_seconds=-1)
assert provider.verify_session(access_token=token) is None
def test_wrong_audience_raises(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, aud="some-other-client")
with pytest.raises(ProviderError, match="verification failed"):
provider.verify_session(access_token=token)
def test_wrong_issuer_raises(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, iss="https://evil.example")
with pytest.raises(ProviderError, match="verification failed"):
provider.verify_session(access_token=token)
def test_failure_message_surfaces_claims(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, iss="https://evil.example")
with pytest.raises(ProviderError) as excinfo:
provider.verify_session(access_token=token)
msg = str(excinfo.value)
assert "'https://evil.example'" in msg
assert f"'{_ISSUER}'" in msg
def test_missing_sub_raises(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, sub="")
with pytest.raises(ProviderError, match="sub"):
provider.verify_session(access_token=token)
def test_display_name_falls_back_to_preferred_username(
self, provider, rsa_keypair
):
token = _mint_id_token(
rsa_keypair,
name=None,
email=None,
extra_claims={"preferred_username": "alice42"},
)
session = provider.verify_session(access_token=token)
assert session is not None
assert session.display_name == "alice42"
def test_org_id_from_org_claim(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, org_id="acme-corp")
session = provider.verify_session(access_token=token)
assert session is not None
assert session.org_id == "acme-corp"
def test_org_id_from_groups_when_no_org_claim(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair, groups=["admins", "users"])
session = provider.verify_session(access_token=token)
assert session is not None
assert session.org_id == "admins,users"
def test_org_id_empty_when_neither_present(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair)
session = provider.verify_session(access_token=token)
assert session is not None
assert session.org_id == ""
def test_jwks_unreachable_raises(self, provider, rsa_keypair):
token = _mint_id_token(rsa_keypair)
bad_client = MagicMock()
bad_client.get_signing_key_from_jwt.side_effect = jwt.PyJWKClientError(
"fetch failed"
)
provider._jwks_client = bad_client
with pytest.raises(ProviderError, match="JWKS"):
provider.verify_session(access_token=token)
# ---------------------------------------------------------------------------
# refresh_session + revoke_session
# ---------------------------------------------------------------------------
class TestRefreshAndRevoke:
@pytest.fixture
def provider(self, rsa_keypair):
return _make_provider(rsa_keypair)
def test_refresh_happy_path_rotates(self, provider, rsa_keypair):
id_token = _mint_id_token(rsa_keypair)
mock_resp = _mock_post(
200,
{
"id_token": id_token,
"token_type": "Bearer",
"refresh_token": "rt_rotated",
},
)
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
) as mock_post:
session = provider.refresh_session(refresh_token="rt_old")
assert isinstance(session, Session)
assert session.access_token == id_token
assert session.refresh_token == "rt_rotated"
assert session.provider == "self-hosted"
_, kwargs = mock_post.call_args
assert kwargs["data"]["grant_type"] == "refresh_token"
assert kwargs["data"]["refresh_token"] == "rt_old"
assert kwargs["data"]["client_id"] == _CLIENT_ID
def test_refresh_keeps_previous_rt_when_idp_omits(self, provider, rsa_keypair):
# Some IDPs don't rotate; keep the caller's existing RT alive.
id_token = _mint_id_token(rsa_keypair)
mock_resp = _mock_post(200, {"id_token": id_token, "token_type": "Bearer"})
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
session = provider.refresh_session(refresh_token="rt_kept")
assert session.refresh_token == "rt_kept"
def test_refresh_400_raises_refresh_expired(self, provider):
mock_resp = _mock_post(400, {"error": "invalid_grant"})
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post", return_value=mock_resp
):
with pytest.raises(RefreshExpiredError, match="invalid_grant"):
provider.refresh_session(refresh_token="rt_dead")
def test_refresh_empty_token_no_network(self, provider):
with patch("plugins.dashboard_auth.self_hosted.httpx.post") as mock_post:
with pytest.raises(RefreshExpiredError):
provider.refresh_session(refresh_token="")
mock_post.assert_not_called()
def test_refresh_network_error_raises_provider_error(self, provider):
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post",
side_effect=httpx.RequestError("boom"),
):
with pytest.raises(ProviderError, match="unreachable"):
provider.refresh_session(refresh_token="rt_x")
def test_revoke_posts_to_revocation_endpoint(self, provider):
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post"
) as mock_post:
provider.revoke_session(refresh_token="rt_x")
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
assert args[0] == f"{_ISSUER}/revoke"
assert kwargs["data"]["token"] == "rt_x"
def test_revoke_empty_token_noop(self, provider):
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post"
) as mock_post:
assert provider.revoke_session(refresh_token="") is None
mock_post.assert_not_called()
def test_revoke_swallows_errors(self, provider):
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post",
side_effect=httpx.RequestError("down"),
):
# Must not raise.
assert provider.revoke_session(refresh_token="rt_x") is None
def test_revoke_noop_when_no_revocation_endpoint(self, provider):
provider._discovery["revocation_endpoint"] = ""
with patch(
"plugins.dashboard_auth.self_hosted.httpx.post"
) as mock_post:
assert provider.revoke_session(refresh_token="rt_x") is None
mock_post.assert_not_called()
# ---------------------------------------------------------------------------
# Plugin entry point: env + config.yaml precedence
# ---------------------------------------------------------------------------
class TestPluginRegister:
@pytest.fixture(autouse=True)
def clear_env(self, monkeypatch):
for var in (
"HERMES_DASHBOARD_OIDC_ISSUER",
"HERMES_DASHBOARD_OIDC_CLIENT_ID",
"HERMES_DASHBOARD_OIDC_SCOPES",
):
monkeypatch.delenv(var, raising=False)
@pytest.fixture
def patch_config(self, monkeypatch):
def _set(oauth_block):
cfg = {}
if oauth_block is not None:
cfg = {"dashboard": {"oauth": oauth_block}}
monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg)
return _set
def test_skips_when_unconfigured(self, patch_config):
patch_config(None)
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
assert "HERMES_DASHBOARD_OIDC_ISSUER" in oidc_plugin.LAST_SKIP_REASON
assert "self_hosted" in oidc_plugin.LAST_SKIP_REASON
def test_skips_when_only_issuer_set(self, patch_config, monkeypatch):
patch_config(None)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_ISSUER", _ISSUER)
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
def test_registers_from_env(self, patch_config, monkeypatch):
patch_config(None)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_ISSUER", _ISSUER)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_CLIENT_ID", _CLIENT_ID)
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert isinstance(registered, oidc_plugin.SelfHostedOIDCProvider)
assert registered._issuer == _ISSUER
assert registered._client_id == _CLIENT_ID
assert registered._scopes == "openid profile email"
assert oidc_plugin.LAST_SKIP_REASON == ""
def test_registers_from_config_yaml(self, patch_config):
patch_config(
{"self_hosted": {"issuer": _ISSUER, "client_id": _CLIENT_ID}}
)
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._issuer == _ISSUER
assert registered._client_id == _CLIENT_ID
def test_env_overrides_config(self, patch_config, monkeypatch):
patch_config(
{
"self_hosted": {
"issuer": "https://config.example",
"client_id": "config-client",
}
}
)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_ISSUER", _ISSUER)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_CLIENT_ID", _CLIENT_ID)
ctx = MagicMock()
oidc_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._issuer == _ISSUER
assert registered._client_id == _CLIENT_ID
def test_empty_env_does_not_shadow_config(self, patch_config, monkeypatch):
patch_config(
{"self_hosted": {"issuer": _ISSUER, "client_id": _CLIENT_ID}}
)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_ISSUER", "")
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_CLIENT_ID", "")
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._issuer == _ISSUER
def test_custom_scopes_from_config(self, patch_config):
patch_config(
{
"self_hosted": {
"issuer": _ISSUER,
"client_id": _CLIENT_ID,
"scopes": "openid email",
}
}
)
ctx = MagicMock()
oidc_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._scopes == "openid email"
def test_config_load_failure_falls_through(self, monkeypatch):
def _broken():
raise OSError("unreadable")
monkeypatch.setattr("hermes_cli.config.load_config", _broken)
ctx = MagicMock()
oidc_plugin.register(ctx) # must not raise
ctx.register_dashboard_auth_provider.assert_not_called()
def test_non_dict_oauth_section_tolerated(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"dashboard": {"oauth": "wrong type"}},
)
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
def test_non_https_issuer_skips_with_reason(self, patch_config, monkeypatch):
patch_config(None)
monkeypatch.setenv(
"HERMES_DASHBOARD_OIDC_ISSUER", "http://insecure.example"
)
monkeypatch.setenv("HERMES_DASHBOARD_OIDC_CLIENT_ID", _CLIENT_ID)
ctx = MagicMock()
oidc_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
assert "construction failed" in oidc_plugin.LAST_SKIP_REASON
tests/plugins/image_gen/__init__.py
View File
tests/plugins/image_gen/check_parity_vs_main.py
+300
View File
@@ -0,0 +1,300 @@
"""Behavior-parity check for the image-gen FAL plugin migration (#26241).
Spawns one subprocess per (version, scenario) cell — pinned to either
``origin/main`` (legacy in-tree FAL fall-through + ``configured == "fal"``
skip in ``_dispatch_to_plugin_provider``) or this PR's worktree (FAL is
itself a plugin and the dispatcher routes every set provider through
the registry). Each subprocess clears all FAL-related env vars + writes
a ``config.yaml``, then asks the dispatcher how it would route an
``image_generate`` call. The emitted shape tuple is
``{dispatch_kind, provider_name, model}``:
* ``dispatch_kind`` ∈ ``{"legacy_fal", "plugin", "error", None}`` —
whether the call would go straight to the in-tree pipeline,
through ``_dispatch_to_plugin_provider``, raise an explicit
provider-not-registered error, or fall through silently.
* ``provider_name`` — when ``dispatch_kind == "plugin"``, the
resolved provider name. ``None`` otherwise.
* ``model`` — the resolved FAL model id when applicable.
The parent process diffs the shapes per scenario. A diff means the
migration introduced an observable behaviour change vs origin/main —
likely a real regression for users on the existing config keys.
Run from the PR worktree:
python tests/plugins/image_gen/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
# Pin one path to current main, one to the PR worktree.
# ``REPO_ROOT`` is ``.../.worktrees/<name>``; the main checkout lives
# two levels up. When running directly from a regular clone (no
# worktree), ``MAIN_DIR`` falls back to a sibling ``hermes-agent-main``
# checkout if one exists.
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "image_generation_tool.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "image_generation_tool.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "image_generation_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear FAL-related env so dispatch decisions are config-driven.
for k in (
"FAL_KEY", "FAL_QUEUE_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
"FAL_IMAGE_MODEL",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
import tools.image_generation_tool as image_tool
dispatch_kind = None
provider_name = None
model = None
error_text = None
try:
raw = image_tool._dispatch_to_plugin_provider("ping", "landscape")
if raw is None:
dispatch_kind = "legacy_fal"
else:
parsed = json.loads(raw) if isinstance(raw, str) else raw
if isinstance(parsed, dict):
if parsed.get("error_type") == "provider_not_registered":
dispatch_kind = "error"
error_text = parsed.get("error")
else:
dispatch_kind = "plugin"
provider_name = parsed.get("provider")
model = parsed.get("model")
else:
dispatch_kind = "unknown_payload"
if model is None:
# _resolve_fal_model still returns the active FAL model id even
# when dispatch goes to a non-FAL plugin — used for the diff
# only when applicable.
try:
model_id, _meta = image_tool._resolve_fal_model()
if dispatch_kind == "legacy_fal":
model = model_id
except Exception:
pass
except Exception as exc:
dispatch_kind = "exception"
error_text = repr(exc)
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"model": model,
"error_present": error_text is not None,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
# (label, config.yaml body, extra env vars)
("no-config-no-env", "", {}),
(
"explicit-fal-no-creds",
"image_gen:\n provider: fal\n",
{},
),
(
"explicit-fal-with-creds",
"image_gen:\n provider: fal\n",
{"FAL_KEY": "test-key"},
),
(
"explicit-fal-with-model",
"image_gen:\n provider: fal\n model: fal-ai/flux-2-pro\n",
{"FAL_KEY": "test-key"},
),
(
"explicit-typo-provider",
"image_gen:\n provider: not-a-real-backend\n",
{"FAL_KEY": "test-key"},
),
(
"managed-gateway-only",
"",
{
"TOOL_GATEWAY_DOMAIN": "nousresearch.com",
"TOOL_GATEWAY_USER_TOKEN": "nous-token",
},
),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
"""Reduce to the parts that matter for user-visible parity.
On origin/main, ``explicit-fal-*`` scenarios short-circuit to
``legacy_fal`` because of the ``configured == "fal"`` skip. On the
PR, those same scenarios route through the plugin and emit
``dispatch_kind == "plugin"`` with ``provider_name == "fal"``.
Both shapes are functionally equivalent — the plugin's ``generate()``
re-enters the same in-tree pipeline via ``_it`` indirection — but
we want the diff to be visible so reviewers can sign off on the
intentional behaviour delta.
"""
return {
"dispatch_kind": shape.get("dispatch_kind"),
"provider_name": shape.get("provider_name"),
"model": shape.get("model"),
"error_present": shape.get("error_present"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "explicit-fal-*" returns legacy_fal; on PR, plugin
# dispatch. That's the only acceptable diff — flag everything
# else as a regression.
legacy_to_plugin_fal = (
main_reduced.get("dispatch_kind") == "legacy_fal"
and pr_reduced.get("dispatch_kind") == "plugin"
and pr_reduced.get("provider_name") == "fal"
)
if legacy_to_plugin_fal:
print(f" [DIFF] {label}: legacy_fal → plugin (fal) — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"legacy_fal → plugin dispatch for explicit FAL paths."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
tests/plugins/image_gen/test_fal_provider.py
+225
View File
@@ -0,0 +1,225 @@
#!/usr/bin/env python3
"""Tests for the FAL.ai image generation plugin.
The plugin is a thin registration adapter — actual FAL pipeline logic
lives in ``tools.image_generation_tool`` and is exercised by
``tests/tools/test_image_generation.py``. These tests focus on:
* the ``ImageGenProvider`` ABC surface (name, models, schema)
* call-time indirection (``_it`` resolution at ``generate()`` time so
``monkeypatch.setattr(image_tool, ...)`` keeps working)
* response shape stamping (provider/prompt/aspect_ratio/model)
"""
from __future__ import annotations
import json
from unittest.mock import MagicMock
# ---------------------------------------------------------------------------
# Provider surface
# ---------------------------------------------------------------------------
class TestFalImageGenProviderSurface:
def test_name(self):
from plugins.image_gen.fal import FalImageGenProvider
assert FalImageGenProvider().name == "fal"
def test_display_name(self):
from plugins.image_gen.fal import FalImageGenProvider
assert FalImageGenProvider().display_name == "FAL.ai"
def test_default_model_matches_legacy(self):
from plugins.image_gen.fal import FalImageGenProvider
from tools.image_generation_tool import DEFAULT_MODEL
assert FalImageGenProvider().default_model() == DEFAULT_MODEL
def test_list_models_uses_legacy_catalog(self):
from plugins.image_gen.fal import FalImageGenProvider
from tools.image_generation_tool import FAL_MODELS
provider = FalImageGenProvider()
models = provider.list_models()
ids = {m["id"] for m in models}
# Whatever FAL_MODELS ships, the provider mirrors verbatim.
assert ids == set(FAL_MODELS.keys())
# Spot-check the expected first-class fields are present.
for entry in models:
for field in ("id", "display", "speed", "strengths", "price"):
assert field in entry
def test_setup_schema_advertises_fal_key(self):
from plugins.image_gen.fal import FalImageGenProvider
schema = FalImageGenProvider().get_setup_schema()
assert schema["name"] == "FAL.ai"
assert schema["badge"] == "paid"
env_keys = {entry["key"] for entry in schema.get("env_vars", [])}
assert "FAL_KEY" in env_keys
class TestFalImageGenProviderAvailability:
def test_is_available_when_legacy_check_passes(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: True)
assert FalImageGenProvider().is_available() is True
def test_is_available_false_when_legacy_check_fails(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: False)
assert FalImageGenProvider().is_available() is False
def test_is_available_handles_legacy_exception(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
def _boom():
raise RuntimeError("config broke")
monkeypatch.setattr(image_tool, "check_fal_api_key", _boom)
# Picker must not propagate exceptions — show as "not available".
assert FalImageGenProvider().is_available() is False
# ---------------------------------------------------------------------------
# generate() — call-time indirection
# ---------------------------------------------------------------------------
class TestFalImageGenProviderGenerate:
def test_generate_delegates_to_legacy_image_generate_tool(self, monkeypatch):
"""Plugin must look up ``image_generate_tool`` at call time so
``monkeypatch.setattr(image_tool, "image_generate_tool", ...)``
takes effect."""
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
captured = {}
def fake_image_generate_tool(prompt, aspect_ratio, **kwargs):
captured["prompt"] = prompt
captured["aspect_ratio"] = aspect_ratio
captured["kwargs"] = kwargs
return json.dumps({"success": True, "image": "https://fake/image.png"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake_image_generate_tool)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
result = FalImageGenProvider().generate(
"a serene mountain landscape",
aspect_ratio="square",
seed=42,
)
assert captured["prompt"] == "a serene mountain landscape"
assert captured["aspect_ratio"] == "square"
assert captured["kwargs"] == {"seed": 42}
assert result["success"] is True
assert result["image"] == "https://fake/image.png"
# Stamped fields for the unified response shape
assert result["provider"] == "fal"
assert result["prompt"] == "a serene mountain landscape"
assert result["aspect_ratio"] == "square"
assert result["model"] == "fal-ai/flux-2/klein/9b"
def test_generate_invalid_aspect_ratio_is_coerced(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
seen_aspect = {}
def fake(prompt, aspect_ratio, **kwargs):
seen_aspect["v"] = aspect_ratio
return json.dumps({"success": True, "image": "x"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
FalImageGenProvider().generate("p", aspect_ratio="not-a-real-ratio")
# ``resolve_aspect_ratio`` clamps to landscape.
assert seen_aspect["v"] == "landscape"
def test_generate_passthrough_drops_none_kwargs(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
seen = {}
def fake(prompt, aspect_ratio, **kwargs):
seen.update(kwargs)
return json.dumps({"success": True, "image": "x"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
FalImageGenProvider().generate(
"p",
aspect_ratio="landscape",
seed=None,
num_images=2,
guidance_scale=None,
)
# ``None`` values must not be forwarded — they'd override the
# model's defaults inside the legacy payload builder.
assert "seed" not in seen
assert "guidance_scale" not in seen
assert seen.get("num_images") == 2
def test_generate_catches_exception_from_legacy(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
def boom(*args, **kwargs):
raise RuntimeError("FAL endpoint exploded")
monkeypatch.setattr(image_tool, "image_generate_tool", boom)
result = FalImageGenProvider().generate("p")
assert result["success"] is False
assert "FAL image generation failed" in result["error"]
assert result["error_type"] == "RuntimeError"
assert result["provider"] == "fal"
def test_generate_invalid_json_response(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "image_generate_tool", lambda **kw: "not-json")
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
result = FalImageGenProvider().generate("p")
assert result["success"] is False
assert "Invalid JSON" in result["error"]
assert result["provider"] == "fal"
# ---------------------------------------------------------------------------
# Registry wiring
# ---------------------------------------------------------------------------
class TestFalImageGenPluginRegistration:
def test_register_wires_provider_into_registry(self):
from plugins.image_gen.fal import FalImageGenProvider, register
ctx = MagicMock()
register(ctx)
ctx.register_image_gen_provider.assert_called_once()
(registered,), _ = ctx.register_image_gen_provider.call_args
assert isinstance(registered, FalImageGenProvider)
tests/plugins/image_gen/test_krea_provider.py
+625
View File
@@ -0,0 +1,625 @@
#!/usr/bin/env python3
"""Tests for Krea image generation provider."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _fake_api_key(monkeypatch):
"""Ensure KREA_API_KEY is set for all tests."""
monkeypatch.setenv("KREA_API_KEY", "test-key-12345")
def _completed_job(url: str = "https://krea.cdn/img.png") -> dict:
return {
"job_id": "00000000-0000-0000-0000-000000000abc",
"status": "completed",
"created_at": "2026-05-27T00:00:00Z",
"completed_at": "2026-05-27T00:00:30Z",
"result": {"urls": [url]},
}
def _submit_response(job_id: str = "00000000-0000-0000-0000-000000000abc"):
resp = MagicMock()
resp.status_code = 200
resp.raise_for_status = MagicMock()
resp.json.return_value = {
"job_id": job_id,
"status": "queued",
"created_at": "2026-05-27T00:00:00Z",
"completed_at": None,
"result": None,
}
return resp
def _poll_response(body: dict):
resp = MagicMock()
resp.status_code = 200
resp.raise_for_status = MagicMock()
resp.json.return_value = body
return resp
# ---------------------------------------------------------------------------
# Provider class tests
# ---------------------------------------------------------------------------
class TestKreaImageGenProvider:
def test_name(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().name == "krea"
def test_display_name(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().display_name == "Krea"
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("KREA_API_KEY", "sk-test")
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().is_available() is True
def test_is_available_without_key(self, monkeypatch):
monkeypatch.delenv("KREA_API_KEY", raising=False)
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().is_available() is False
def test_list_models(self):
from plugins.image_gen.krea import KreaImageGenProvider
models = KreaImageGenProvider().list_models()
ids = {m["id"] for m in models}
assert {"krea-2-medium", "krea-2-large"} <= ids
# Each entry carries the picker fields the registry expects.
for m in models:
assert m["display"]
assert m["speed"]
assert m["strengths"]
assert m["price"]
def test_default_model_is_medium(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().default_model() == "krea-2-medium"
def test_get_setup_schema(self):
from plugins.image_gen.krea import KreaImageGenProvider
schema = KreaImageGenProvider().get_setup_schema()
assert schema["name"] == "Krea"
assert schema["badge"] == "paid"
env_vars = schema["env_vars"]
assert len(env_vars) == 1
assert env_vars[0]["key"] == "KREA_API_KEY"
assert "krea.ai" in env_vars[0]["url"]
# ---------------------------------------------------------------------------
# Model resolution
# ---------------------------------------------------------------------------
class TestModelResolution:
def test_default(self):
from plugins.image_gen.krea import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "krea-2-medium"
assert meta["path"] == "medium"
def test_env_override_large(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
from plugins.image_gen.krea import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "krea-2-large"
assert meta["path"] == "large"
def test_env_override_unknown_falls_back_to_default(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-xxl-fake")
from plugins.image_gen.krea import _resolve_model
model_id, _ = _resolve_model()
assert model_id == "krea-2-medium"
def test_creativity_default(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity(None) == "medium"
def test_creativity_valid(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity("HIGH") == "high"
assert _resolve_creativity(" raw ") == "raw"
def test_creativity_invalid(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity("ultra") == "medium"
# ---------------------------------------------------------------------------
# Generate — main flow
# ---------------------------------------------------------------------------
class TestGenerate:
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("KREA_API_KEY", raising=False)
from plugins.image_gen.krea import KreaImageGenProvider
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert "KREA_API_KEY" in result["error"]
assert result["error_type"] == "auth_required"
def test_empty_prompt(self):
from plugins.image_gen.krea import KreaImageGenProvider
result = KreaImageGenProvider().generate(prompt=" ")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_successful_generation(self):
"""Happy path: submit → one poll → completed → URL downloaded."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job("https://krea.cdn/result.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/krea_krea-2-medium_test.png"),
) as mock_save, \
patch("plugins.image_gen.krea.time.sleep"): # skip real waits
result = KreaImageGenProvider().generate(prompt="A cinematic lamp")
assert result["success"] is True
assert result["image"] == "/tmp/krea_krea-2-medium_test.png"
assert result["provider"] == "krea"
assert result["model"] == "krea-2-medium"
assert result["aspect_ratio"] == "landscape"
assert result["job_id"] == "00000000-0000-0000-0000-000000000abc"
assert result["resolution"] == "1K"
assert result["creativity"] == "medium"
# Submit hit the medium endpoint
post_url = mock_post.call_args[0][0]
assert post_url.endswith("/generate/image/krea/krea-2/medium")
# Poll hit /jobs/{job_id}
poll_url = mock_get.call_args[0][0]
assert "/jobs/00000000-0000-0000-0000-000000000abc" in poll_url
# URL was materialised once
mock_save.assert_called_once()
def test_large_model_routes_to_large_endpoint(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test")
post_url = mock_post.call_args[0][0]
assert post_url.endswith("/generate/image/krea/krea-2/large")
def test_aspect_ratio_mapping(self):
"""Hermes 'square' must map to Krea '1:1' in the wire payload."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test", aspect_ratio="square")
payload = mock_post.call_args.kwargs["json"]
assert payload["aspect_ratio"] == "1:1"
assert payload["resolution"] == "1K"
def test_auth_header(self):
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test")
headers = mock_post.call_args.kwargs["headers"]
assert headers["Authorization"] == "Bearer test-key-12345"
assert headers["Content-Type"] == "application/json"
def test_passthrough_seed_styles_moodboards(self):
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(
prompt="test",
seed=42,
styles=[{"id": "lora-1", "strength": 0.7}],
moodboards=[{"url": "https://x.com/mood.png"}, {"url": "https://x.com/mood2.png"}],
image_style_references=[{"url": f"https://x.com/{i}.png"} for i in range(15)],
creativity="high",
)
payload = mock_post.call_args.kwargs["json"]
assert payload["seed"] == 42
assert payload["styles"] == [{"id": "lora-1", "strength": 0.7}]
assert len(payload["moodboards"]) == 1 # capped at 1
assert len(payload["image_style_references"]) == 10 # capped at 10
assert payload["creativity"] == "high"
def test_unknown_kwargs_ignored(self):
"""Forward-compat: unknown kwargs must not break generate()."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(
prompt="test",
fictional_param="should be ignored",
num_images=4,
)
assert result["success"] is True
# ---------------------------------------------------------------------------
# Generate — error paths
# ---------------------------------------------------------------------------
class TestGenerateErrors:
def test_submit_http_error(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
resp = req_lib.Response()
resp.status_code = 401
resp._content = b'{"error": {"message": "Invalid API key"}}'
resp.headers["Content-Type"] = "application/json"
resp.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=resp)
)
with patch("plugins.image_gen.krea.requests.post", return_value=resp):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "401" in result["error"]
assert "Invalid API key" in result["error"]
def test_submit_timeout(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
with patch(
"plugins.image_gen.krea.requests.post", side_effect=req_lib.Timeout()
):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "timeout"
def test_submit_connection_error(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
with patch(
"plugins.image_gen.krea.requests.post",
side_effect=req_lib.ConnectionError("dns nope"),
):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "connection_error"
def test_submit_missing_job_id(self):
from plugins.image_gen.krea import KreaImageGenProvider
bad_submit = MagicMock()
bad_submit.status_code = 200
bad_submit.raise_for_status = MagicMock()
bad_submit.json.return_value = {"status": "queued"}
with patch("plugins.image_gen.krea.requests.post", return_value=bad_submit):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "invalid_response"
assert "job_id" in result["error"]
def test_job_failed(self):
from plugins.image_gen.krea import KreaImageGenProvider
failed = {
"job_id": "abc",
"status": "failed",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"error": "NSFW content"},
}
submit = _submit_response()
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(failed),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "NSFW" in result["error"]
def test_job_cancelled(self):
from plugins.image_gen.krea import KreaImageGenProvider
cancelled = {
"job_id": "abc",
"status": "cancelled",
"completed_at": "2026-05-27T00:01:00Z",
"result": {},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(cancelled),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "cancelled"
def test_completed_but_missing_urls(self):
from plugins.image_gen.krea import KreaImageGenProvider
completed_empty = {
"job_id": "abc",
"status": "completed",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"urls": []},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(completed_empty),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_url_download_failure_falls_back_to_bare_url(self):
"""Mirror of xAI behaviour — if local cache fails, return the URL."""
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
url = "https://krea.cdn/expired-soon.png"
submit = _submit_response()
poll = _poll_response(_completed_job(url))
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
side_effect=req_lib.HTTPError("404"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert result["image"] == url
def test_polling_picks_up_completed_at_with_unknown_status(self):
"""``completed_at`` set + unrecognised pending status → still terminal."""
from plugins.image_gen.krea import KreaImageGenProvider
# Use a status value that is NOT in our terminal set ("intermediate-complete")
# but with completed_at populated — Krea's spec says completed_at is the
# canonical terminal marker.
oddball = {
"job_id": "abc",
"status": "intermediate-complete",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"urls": ["https://krea.cdn/done.png"]},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(oddball),
), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
class TestPollRetryPolicy:
"""Polling fail-fast on permanent 4xx, retry on transient 5xx/429."""
def _http_error_response(self, status: int):
import requests as req_lib
resp = req_lib.Response()
resp.status_code = status
resp._content = b'{"error": "boom"}'
resp.headers["Content-Type"] = "application/json"
resp.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=resp)
)
return resp
def test_poll_fails_fast_on_401(self):
"""Auth failure mid-poll should not wait the 180s deadline."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(401)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "401" in result["error"]
# One call — no retry on permanent auth failure.
assert mock_get.call_count == 1
def test_poll_fails_fast_on_404(self):
"""Missing job (404) should surface immediately, not retry for 180s."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(404)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "404" in result["error"]
assert mock_get.call_count == 1
def test_poll_fails_fast_on_403(self):
"""Billing/permission failure (403) should not retry."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(403)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert mock_get.call_count == 1
def test_poll_retries_on_503_then_succeeds(self):
"""Transient 5xx should retry and eventually surface a completion."""
from plugins.image_gen.krea import KreaImageGenProvider
flaky = self._http_error_response(503)
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
side_effect=[flaky, flaky, good],
) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert mock_get.call_count == 3
def test_poll_retries_on_429(self):
"""Rate-limit (429) is in the retryable set."""
from plugins.image_gen.krea import KreaImageGenProvider
rate_limited = self._http_error_response(429)
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
side_effect=[rate_limited, good],
) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert mock_get.call_count == 2
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
class TestRegistration:
def test_register(self):
from plugins.image_gen.krea import KreaImageGenProvider, register
mock_ctx = MagicMock()
register(mock_ctx)
mock_ctx.register_image_gen_provider.assert_called_once()
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
assert isinstance(provider, KreaImageGenProvider)
assert provider.name == "krea"
tests/plugins/image_gen/test_openai_codex_provider.py
+236
View File
@@ -0,0 +1,236 @@
"""Tests for the bundled ``openai-codex`` image_gen plugin.
Mirrors ``test_openai_provider.py`` but targets the standalone
Codex/ChatGPT-OAuth-backed provider that uses the Responses
``image_generation`` tool path instead of the ``images.generate`` REST
endpoint.
"""
from __future__ import annotations
import importlib
from pathlib import Path
import pytest
# The plugin directory uses a hyphen, which is not a valid Python identifier
# for the dotted-import form. Load it via importlib so tests don't need to
# touch sys.path or rename the directory.
codex_plugin = importlib.import_module("plugins.image_gen.openai-codex")
# 1×1 transparent PNG — valid bytes for save_b64_image()
_PNG_HEX = (
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
"ae426082"
)
def _b64_png() -> str:
import base64
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
@pytest.fixture(autouse=True)
def _tmp_hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
@pytest.fixture
def provider(monkeypatch):
# Codex plugin is API-key-independent; clear it to make the test honest.
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
return codex_plugin.OpenAICodexImageGenProvider()
# ── Metadata ────────────────────────────────────────────────────────────────
class TestMetadata:
def test_name(self, provider):
assert provider.name == "openai-codex"
def test_display_name(self, provider):
assert provider.display_name == "OpenAI (Codex auth)"
def test_default_model(self, provider):
assert provider.default_model() == "gpt-image-2-medium"
def test_list_models_three_tiers(self, provider):
ids = [m["id"] for m in provider.list_models()]
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
def test_setup_schema_has_no_required_env_vars(self, provider):
schema = provider.get_setup_schema()
assert schema["env_vars"] == []
assert schema["badge"] == "free"
# ── Availability ────────────────────────────────────────────────────────────
class TestAvailability:
def test_unavailable_without_codex_token(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
def test_available_with_codex_token(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is True
def test_openai_api_key_alone_is_not_enough(self, monkeypatch):
# Codex plugin is intentionally orthogonal to the API-key plugin —
# the API key alone must NOT make it appear available.
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
# ── Generate ────────────────────────────────────────────────────────────────
class TestGenerate:
def test_returns_auth_error_without_codex_token(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_returns_invalid_argument_for_empty_prompt(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
result = provider.generate(" ")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_generate_uses_codex_stream_path(self, provider, monkeypatch, tmp_path):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: _b64_png())
result = provider.generate("a cat", aspect_ratio="landscape")
assert result["success"] is True
assert result["model"] == "gpt-image-2-medium"
assert result["provider"] == "openai-codex"
assert result["quality"] == "medium"
saved = Path(result["image"])
assert saved.exists()
assert saved.parent == tmp_path / "cache" / "images"
# Filename prefix differs from the API-key plugin so cache audits can
# tell the two backends apart.
assert saved.name.startswith("openai_codex_")
def test_codex_stream_request_shape(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
captured = {}
def _collect(token, *, prompt, size, quality):
captured.update(codex_plugin._build_responses_payload(
prompt=prompt,
size=size,
quality=quality,
))
return _b64_png()
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _collect)
result = provider.generate("a cat", aspect_ratio="portrait")
assert result["success"] is True
assert captured["model"] == "gpt-5.5"
assert captured["store"] is False
assert captured["input"][0]["type"] == "message"
assert captured["input"][0]["role"] == "user"
assert captured["input"][0]["content"][0]["type"] == "input_text"
assert captured["tool_choice"]["type"] == "allowed_tools"
assert captured["tool_choice"]["mode"] == "required"
assert captured["tool_choice"]["tools"] == [{"type": "image_generation"}]
tool = captured["tools"][0]
assert tool["type"] == "image_generation"
assert tool["model"] == "gpt-image-2"
assert tool["quality"] == "medium"
assert tool["size"] == "1024x1536"
assert tool["output_format"] == "png"
assert tool["background"] == "opaque"
assert tool["partial_images"] == 1
def test_partial_image_event_used_when_done_missing(self):
"""If output_item.done is missing, partial_image_b64 is accepted."""
payload = {
"type": "response.image_generation_call.partial_image",
"partial_image_b64": _b64_png(),
}
assert codex_plugin._extract_image_b64(payload) == _b64_png()
def test_sse_parser_handles_event_and_data_lines(self):
class _Response:
def iter_lines(self):
return iter([
"event: response.output_item.done",
'data: {"item": {"type": "image_generation_call", "result": "abc"}}',
"",
])
events = list(codex_plugin._iter_sse_json(_Response()))
assert events == [{
"type": "response.output_item.done",
"item": {"type": "image_generation_call", "result": "abc"},
}]
def test_final_response_sweep_recovers_image(self):
"""Completed response output is found by recursive payload scanning."""
payload = {
"type": "response.completed",
"response": {
"output": [{
"type": "image_generation_call",
"status": "completed",
"id": "ig_final",
"result": _b64_png(),
}],
},
}
assert codex_plugin._extract_image_b64(payload) == _b64_png()
def test_empty_response_returns_error(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: None)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_stream_exception_returns_api_error(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
def _boom(*args, **kwargs):
raise RuntimeError("cloudflare 403")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _boom)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "cloudflare 403" in result["error"]
# ── Plugin entry point ──────────────────────────────────────────────────────
class TestRegistration:
def test_register_calls_register_image_gen_provider(self):
registered = []
class _Ctx:
def register_image_gen_provider(self, prov):
registered.append(prov)
codex_plugin.register(_Ctx())
assert len(registered) == 1
assert registered[0].name == "openai-codex"
tests/plugins/image_gen/test_openai_provider.py
+271
View File
@@ -0,0 +1,271 @@
"""Tests for the bundled OpenAI image_gen plugin (gpt-image-2, three tiers)."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import plugins.image_gen.openai as openai_plugin
# 1×1 transparent PNG — valid bytes for save_b64_image()
_PNG_HEX = (
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
"ae426082"
)
def _b64_png() -> str:
import base64
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
def _fake_response(*, b64=None, url=None, revised_prompt=None):
item = SimpleNamespace(b64_json=b64, url=url, revised_prompt=revised_prompt)
return SimpleNamespace(data=[item])
@pytest.fixture(autouse=True)
def _tmp_hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
@pytest.fixture
def provider(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
return openai_plugin.OpenAIImageGenProvider()
def _patched_openai(fake_client: MagicMock):
fake_openai = MagicMock()
fake_openai.OpenAI.return_value = fake_client
return patch.dict("sys.modules", {"openai": fake_openai})
# ── Metadata ────────────────────────────────────────────────────────────────
class TestMetadata:
def test_name(self, provider):
assert provider.name == "openai"
def test_default_model(self, provider):
assert provider.default_model() == "gpt-image-2-medium"
def test_list_models_three_tiers(self, provider):
ids = [m["id"] for m in provider.list_models()]
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
def test_catalog_entries_have_display_speed_strengths(self, provider):
for entry in provider.list_models():
assert entry["display"].startswith("GPT Image 2")
assert entry["speed"]
assert entry["strengths"]
# ── Availability ────────────────────────────────────────────────────────────
class TestAvailability:
def test_no_api_key_unavailable(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
assert openai_plugin.OpenAIImageGenProvider().is_available() is False
def test_api_key_set_available(self, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test")
assert openai_plugin.OpenAIImageGenProvider().is_available() is True
# ── Model resolution ────────────────────────────────────────────────────────
class TestModelResolution:
def test_default_is_medium(self):
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-medium"
assert meta["quality"] == "medium"
def test_env_var_override(self, monkeypatch):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "gpt-image-2-high")
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-high"
assert meta["quality"] == "high"
def test_env_var_unknown_falls_back(self, monkeypatch):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "bogus-tier")
model_id, _ = openai_plugin._resolve_model()
assert model_id == openai_plugin.DEFAULT_MODEL
def test_config_openai_model(self, tmp_path):
import yaml
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"openai": {"model": "gpt-image-2-low"}}})
)
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-low"
assert meta["quality"] == "low"
def test_config_top_level_model(self, tmp_path):
"""``image_gen.model: gpt-image-2-high`` also works (top-level)."""
import yaml
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"model": "gpt-image-2-high"}})
)
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-high"
assert meta["quality"] == "high"
# ── Generate ────────────────────────────────────────────────────────────────
class TestGenerate:
def test_empty_prompt_rejected(self, provider):
result = provider.generate("", aspect_ratio="square")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
result = openai_plugin.OpenAIImageGenProvider().generate("a cat")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_b64_saves_to_cache(self, provider, tmp_path):
png_bytes = bytes.fromhex(_PNG_HEX)
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
result = provider.generate("a cat", aspect_ratio="landscape")
assert result["success"] is True
assert result["model"] == "gpt-image-2-medium"
assert result["aspect_ratio"] == "landscape"
assert result["provider"] == "openai"
assert result["quality"] == "medium"
saved = Path(result["image"])
assert saved.exists()
assert saved.parent == tmp_path / "cache" / "images"
assert saved.read_bytes() == png_bytes
call_kwargs = fake_client.images.generate.call_args.kwargs
# All tiers hit the single underlying API model.
assert call_kwargs["model"] == "gpt-image-2"
assert call_kwargs["quality"] == "medium"
assert call_kwargs["size"] == "1536x1024"
# gpt-image-2 rejects response_format — we must NOT send it.
assert "response_format" not in call_kwargs
@pytest.mark.parametrize("tier,expected_quality", [
("gpt-image-2-low", "low"),
("gpt-image-2-medium", "medium"),
("gpt-image-2-high", "high"),
])
def test_tier_maps_to_quality(self, provider, monkeypatch, tier, expected_quality):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", tier)
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["model"] == tier
assert result["quality"] == expected_quality
assert fake_client.images.generate.call_args.kwargs["quality"] == expected_quality
# Always the same underlying API model regardless of tier.
assert fake_client.images.generate.call_args.kwargs["model"] == "gpt-image-2"
@pytest.mark.parametrize("aspect,expected_size", [
("landscape", "1536x1024"),
("square", "1024x1024"),
("portrait", "1024x1536"),
])
def test_aspect_ratio_mapping(self, provider, aspect, expected_size):
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
provider.generate("a cat", aspect_ratio=aspect)
assert fake_client.images.generate.call_args.kwargs["size"] == expected_size
def test_revised_prompt_passed_through(self, provider):
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=_b64_png(), revised_prompt="A photo of a cat",
)
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["revised_prompt"] == "A photo of a cat"
def test_api_error_returns_error_response(self, provider):
fake_client = MagicMock()
fake_client.images.generate.side_effect = RuntimeError("boom")
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "boom" in result["error"]
def test_empty_response_data(self, provider):
fake_client = MagicMock()
fake_client.images.generate.return_value = SimpleNamespace(data=[])
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_url_response_is_cached_locally(self, provider):
"""OpenAI URL response (if API ever returns one) is cached locally.
Pre-fix this asserted the bare URL passed through; symmetric to the
xAI #26942 fix. Even though gpt-image-2 returns b64 today, every
``image_gen`` provider must guarantee the gateway gets a stable
file path so ephemeral signed URLs can't expire mid-flight.
"""
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=None, url="https://example.com/img.png",
)
with _patched_openai(fake_client), patch(
"plugins.image_gen.openai.save_url_image",
return_value=Path("/tmp/openai_gpt-image-2_20260524_000000_deadbeef.png"),
) as mock_save_url:
result = provider.generate("a cat")
assert result["success"] is True
assert result["image"].startswith("/")
assert "example.com" not in result["image"]
mock_save_url.assert_called_once()
def test_url_response_falls_back_to_bare_url_when_download_fails(self, provider):
"""Cache failure must not turn into a tool error — symmetric with xAI."""
import requests as req_lib
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=None, url="https://example.com/img.png",
)
with _patched_openai(fake_client), patch(
"plugins.image_gen.openai.save_url_image",
side_effect=req_lib.HTTPError("404 from CDN"),
):
result = provider.generate("a cat")
assert result["success"] is True
assert result["image"] == "https://example.com/img.png"
tests/plugins/image_gen/test_xai_provider.py
+336
View File
@@ -0,0 +1,336 @@
#!/usr/bin/env python3
"""Tests for xAI image generation provider."""
from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _fake_api_key(monkeypatch):
"""Ensure XAI_API_KEY is set for all tests."""
monkeypatch.setenv("XAI_API_KEY", "test-key-12345")
# ---------------------------------------------------------------------------
# Provider class tests
# ---------------------------------------------------------------------------
class TestXAIImageGenProvider:
def test_name(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.name == "xai"
def test_display_name(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.display_name == "xAI (Grok)"
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("XAI_API_KEY", "sk-xxx")
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.is_available() is True
def test_is_available_without_key(self, monkeypatch):
monkeypatch.delenv("XAI_API_KEY", raising=False)
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.is_available() is False
def test_list_models(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
models = provider.list_models()
assert len(models) >= 1
assert models[0]["id"] == "grok-imagine-image"
def test_default_model(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.default_model() == "grok-imagine-image"
def test_get_setup_schema(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
schema = provider.get_setup_schema()
assert schema["name"] == "xAI Grok Imagine (image)"
assert schema["badge"] == "paid"
# Auth resolution is delegated to the shared "xai_grok" post_setup
# hook so the picker doesn't blindly prompt for XAI_API_KEY when the
# user is already signed in via xAI Grok OAuth.
assert schema["env_vars"] == []
assert schema["post_setup"] == "xai_grok"
# ---------------------------------------------------------------------------
# Config tests
# ---------------------------------------------------------------------------
class TestConfig:
def test_default_model(self):
from plugins.image_gen.xai import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "grok-imagine-image"
def test_default_resolution(self):
from plugins.image_gen.xai import _resolve_resolution
assert _resolve_resolution() == "1k"
def test_custom_model(self, monkeypatch):
monkeypatch.setenv("XAI_IMAGE_MODEL", "grok-imagine-image")
from plugins.image_gen.xai import _resolve_model
model_id, _ = _resolve_model()
assert model_id == "grok-imagine-image"
# ---------------------------------------------------------------------------
# Generate tests
# ---------------------------------------------------------------------------
class TestGenerate:
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("XAI_API_KEY", raising=False)
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert "XAI_API_KEY" in result["error"]
def test_successful_generation(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"b64_json": "dGVzdC1pbWFnZS1kYXRh"}], # base64 "test-image-data"
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
with patch("plugins.image_gen.xai.save_b64_image", return_value="/tmp/test.png"):
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True
assert result["image"] == "/tmp/test.png"
assert result["provider"] == "xai"
assert result["model"] == "grok-imagine-image"
def test_successful_url_response(self):
"""xAI URL response is cached locally — #26942 contract.
Pre-fix this asserted ``result["image"] == "<the bare URL>"``, which
was exactly the bug: xAI's ``imgen.x.ai/xai-tmp-*`` URLs expire fast
and the gateway 404'd by ``send_photo`` time. Post-fix the URL
bytes are downloaded at tool-completion and the result carries an
absolute filesystem path the gateway can upload from.
"""
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
patch(
"plugins.image_gen.xai.save_url_image",
return_value=Path("/tmp/xai_grok-imagine-image_20260524_000000_deadbeef.jpg"),
) as mock_save_url:
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True
assert result["image"].startswith("/"), (
f"URL response must be cached to an absolute path, got {result['image']!r}"
)
assert "imgen.x.ai" not in result["image"], (
"ephemeral xAI URL must not leak into result.image — caller will 404"
)
# The downloader should have been called exactly once with the URL
# and an xai-prefixed cache filename.
mock_save_url.assert_called_once()
call_args, call_kwargs = mock_save_url.call_args
assert call_args[0] == "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"
assert call_kwargs.get("prefix", "").startswith("xai_")
def test_url_response_falls_back_to_bare_url_when_download_fails(self):
"""If caching the URL fails (network blip, 404 in-flight), the
provider must NOT hard-error — fall through to returning the bare
URL so the agent surface at least sees *something*. The gateway's
existing URL-send fallback then has a chance to succeed; if it
too 404s, the user gets the original (now legible) error rather
than an opaque "image generation failed" tool result.
"""
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
patch(
"plugins.image_gen.xai.save_url_image",
side_effect=req_lib.HTTPError("404 from CDN"),
):
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True, (
"Cache failure must not turn into a tool error — gateway gets a chance to retry"
)
assert result["image"] == "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"
def test_api_error(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 401
mock_resp.text = "Unauthorized"
mock_resp.json.return_value = {"error": {"message": "Invalid API key"}}
mock_resp.raise_for_status.side_effect = req_lib.HTTPError(response=mock_resp)
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
def test_api_error_preserves_real_response_status(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
response = req_lib.Response()
response.status_code = 401
response._content = json.dumps({"error": {"message": "Invalid API key"}}).encode()
response.headers["Content-Type"] = "application/json"
response.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=response)
)
with patch("plugins.image_gen.xai.requests.post", return_value=response):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "xAI image generation failed (401): Invalid API key" in result["error"]
def test_timeout(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
with patch("plugins.image_gen.xai.requests.post", side_effect=req_lib.Timeout()):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "timeout"
def test_empty_response(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {"data": []}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_auth_header(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://xai.image/test.png"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
provider = XAIImageGenProvider()
provider.generate(prompt="test")
call_args = mock_post.call_args
headers = call_args.kwargs.get("headers") or call_args[1].get("headers")
assert "Bearer test-key-12345" in headers["Authorization"]
assert "Hermes-Agent" in headers["User-Agent"]
def test_payload_resolution_is_literal_1k_or_2k(self):
"""Regression: xAI API rejects numeric resolutions ("1024"/"2048") with 422.
The endpoint expects the literal strings "1k" or "2k". Ensure the wire
payload carries that literal — not a numeric mapping. See PR #18678.
"""
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {"data": [{"url": "https://xai.image/test.png"}]}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
provider = XAIImageGenProvider()
provider.generate(prompt="test")
payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json")
assert payload["resolution"] in {"1k", "2k"}, (
f"resolution must be the literal '1k' or '2k', got {payload['resolution']!r}"
)
# ---------------------------------------------------------------------------
# Registration test
# ---------------------------------------------------------------------------
class TestRegistration:
def test_register(self):
from plugins.image_gen.xai import XAIImageGenProvider, register
mock_ctx = MagicMock()
register(mock_ctx)
mock_ctx.register_image_gen_provider.assert_called_once()
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
assert isinstance(provider, XAIImageGenProvider)
assert provider.name == "xai"
tests/plugins/memory/__init__.py
View File
tests/plugins/memory/test_hindsight_provider.py
+1631
View File
File diff suppressed because it is too large Load Diff
tests/plugins/memory/test_mem0_v2.py
+241
View File
@@ -0,0 +1,241 @@
"""Tests for Mem0 API v2 compatibility — filters param and dict response unwrapping.
Salvaged from PRs #5301 (qaqcvc) and #5117 (vvvanguards).
"""
import json
import os
import stat
import pytest
from plugins.memory.mem0 import Mem0MemoryProvider
class FakeClientV2:
"""Fake Mem0 client that returns v2-style dict responses and captures call kwargs."""
def __init__(self, search_results=None, all_results=None):
self._search_results = search_results or {"results": []}
self._all_results = all_results or {"results": []}
self.captured_search = {}
self.captured_get_all = {}
self.captured_add = []
def search(self, **kwargs):
self.captured_search = kwargs
return self._search_results
def get_all(self, **kwargs):
self.captured_get_all = kwargs
return self._all_results
def add(self, messages, **kwargs):
self.captured_add.append({"messages": messages, **kwargs})
# ---------------------------------------------------------------------------
# Filter migration: bare user_id= -> filters={}
# ---------------------------------------------------------------------------
class TestMem0FiltersV2:
"""All API calls must use filters={} instead of bare user_id= kwargs."""
def _make_provider(self, monkeypatch, client):
provider = Mem0MemoryProvider()
provider.initialize("test-session")
provider._user_id = "u123"
provider._agent_id = "hermes"
monkeypatch.setattr(provider, "_get_client", lambda: client)
return provider
def test_search_uses_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.handle_tool_call("mem0_search", {"query": "hello", "top_k": 3, "rerank": False})
assert client.captured_search["query"] == "hello"
assert client.captured_search["top_k"] == 3
assert client.captured_search["rerank"] is False
assert client.captured_search["filters"] == {"user_id": "u123"}
# Must NOT have bare user_id kwarg
assert "user_id" not in {k for k in client.captured_search if k != "filters"}
def test_profile_uses_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.handle_tool_call("mem0_profile", {})
assert client.captured_get_all["filters"] == {"user_id": "u123"}
assert "user_id" not in {k for k in client.captured_get_all if k != "filters"}
def test_prefetch_uses_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.queue_prefetch("hello")
provider._prefetch_thread.join(timeout=2)
assert client.captured_search["query"] == "hello"
assert client.captured_search["filters"] == {"user_id": "u123"}
assert "user_id" not in {k for k in client.captured_search if k != "filters"}
def test_sync_turn_uses_write_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.sync_turn("user said this", "assistant replied", session_id="s1")
provider._sync_thread.join(timeout=2)
assert len(client.captured_add) == 1
call = client.captured_add[0]
assert call["user_id"] == "u123"
assert call["agent_id"] == "hermes"
def test_conclude_uses_write_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.handle_tool_call("mem0_conclude", {"conclusion": "user likes dark mode"})
assert len(client.captured_add) == 1
call = client.captured_add[0]
assert call["user_id"] == "u123"
assert call["agent_id"] == "hermes"
assert call["infer"] is False
def test_read_filters_no_agent_id(self):
"""Read filters should use user_id only — cross-session recall across agents."""
provider = Mem0MemoryProvider()
provider._user_id = "u123"
provider._agent_id = "hermes"
assert provider._read_filters() == {"user_id": "u123"}
def test_write_filters_include_agent_id(self):
"""Write filters should include agent_id for attribution."""
provider = Mem0MemoryProvider()
provider._user_id = "u123"
provider._agent_id = "hermes"
assert provider._write_filters() == {"user_id": "u123", "agent_id": "hermes"}
# ---------------------------------------------------------------------------
# Dict response unwrapping (API v2 wraps in {"results": [...]})
# ---------------------------------------------------------------------------
class TestMem0ResponseUnwrapping:
"""API v2 returns {"results": [...]} dicts; we must extract the list."""
def _make_provider(self, monkeypatch, client):
provider = Mem0MemoryProvider()
provider.initialize("test-session")
monkeypatch.setattr(provider, "_get_client", lambda: client)
return provider
def test_profile_dict_response(self, monkeypatch):
client = FakeClientV2(all_results={"results": [{"memory": "alpha"}, {"memory": "beta"}]})
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call("mem0_profile", {}))
assert result["count"] == 2
assert "alpha" in result["result"]
assert "beta" in result["result"]
def test_profile_list_response_backward_compat(self, monkeypatch):
"""Old API returned bare lists — still works."""
client = FakeClientV2(all_results=[{"memory": "gamma"}])
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call("mem0_profile", {}))
assert result["count"] == 1
assert "gamma" in result["result"]
def test_search_dict_response(self, monkeypatch):
client = FakeClientV2(search_results={
"results": [{"memory": "foo", "score": 0.9}, {"memory": "bar", "score": 0.7}]
})
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call(
"mem0_search", {"query": "test", "top_k": 5}
))
assert result["count"] == 2
assert result["results"][0]["memory"] == "foo"
def test_search_list_response_backward_compat(self, monkeypatch):
"""Old API returned bare lists — still works."""
client = FakeClientV2(search_results=[{"memory": "baz", "score": 0.8}])
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call(
"mem0_search", {"query": "test"}
))
assert result["count"] == 1
def test_unwrap_results_edge_cases(self):
"""_unwrap_results handles all shapes gracefully."""
assert Mem0MemoryProvider._unwrap_results({"results": [1, 2]}) == [1, 2]
assert Mem0MemoryProvider._unwrap_results([3, 4]) == [3, 4]
assert Mem0MemoryProvider._unwrap_results({}) == []
assert Mem0MemoryProvider._unwrap_results(None) == []
assert Mem0MemoryProvider._unwrap_results("unexpected") == []
def test_prefetch_dict_response(self, monkeypatch):
client = FakeClientV2(search_results={
"results": [{"memory": "user prefers dark mode"}]
})
provider = Mem0MemoryProvider()
provider.initialize("test-session")
monkeypatch.setattr(provider, "_get_client", lambda: client)
provider.queue_prefetch("preferences")
provider._prefetch_thread.join(timeout=2)
result = provider.prefetch("preferences")
assert "dark mode" in result
# ---------------------------------------------------------------------------
# Default preservation
# ---------------------------------------------------------------------------
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode bits not enforced on Windows")
def test_save_config_sets_owner_only_permissions(tmp_path):
"""mem0.json must be written with 0o600 so API key is not world-readable."""
provider = Mem0MemoryProvider()
provider.save_config({"api_key": "m0-test-key"}, str(tmp_path))
config_file = tmp_path / "mem0.json"
assert config_file.exists()
mode = stat.S_IMODE(config_file.stat().st_mode)
assert mode == 0o600, f"Expected 0o600 (owner-only), got {oct(mode)}"
class TestMem0Defaults:
"""Ensure we don't break existing users' defaults."""
def test_default_user_id_hermes_user(self, monkeypatch, tmp_path):
monkeypatch.setenv("MEM0_API_KEY", "test-key")
monkeypatch.delenv("MEM0_USER_ID", raising=False)
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
provider = Mem0MemoryProvider()
provider.initialize("test")
assert provider._user_id == "hermes-user"
def test_default_agent_id_hermes(self, monkeypatch, tmp_path):
monkeypatch.setenv("MEM0_API_KEY", "test-key")
monkeypatch.delenv("MEM0_AGENT_ID", raising=False)
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
provider = Mem0MemoryProvider()
provider.initialize("test")
assert provider._agent_id == "hermes"
tests/plugins/memory/test_openviking_provider.py
+422
View File
@@ -0,0 +1,422 @@
import json
import zipfile
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from plugins.memory.openviking import OpenVikingMemoryProvider, _VikingClient
def test_tool_search_sorts_by_raw_score_across_buckets():
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"result": {
"memories": [
{"uri": "viking://memories/1", "score": 0.9003, "abstract": "memory result"},
],
"resources": [
{"uri": "viking://resources/1", "score": 0.9004, "abstract": "resource result"},
],
"skills": [
{"uri": "viking://skills/1", "score": 0.8999, "abstract": "skill result"},
],
"total": 3,
}
}
result = json.loads(provider._tool_search({"query": "ranking"}))
assert [entry["uri"] for entry in result["results"]] == [
"viking://resources/1",
"viking://memories/1",
"viking://skills/1",
]
assert [entry["score"] for entry in result["results"]] == [0.9, 0.9, 0.9]
assert result["total"] == 3
def test_tool_search_sorts_missing_raw_score_after_negative_scores():
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"result": {
"memories": [
{"uri": "viking://memories/missing", "abstract": "missing score"},
],
"resources": [
{"uri": "viking://resources/negative", "score": -0.25, "abstract": "negative score"},
],
"skills": [
{"uri": "viking://skills/positive", "score": 0.1, "abstract": "positive score"},
],
"total": 3,
}
}
result = json.loads(provider._tool_search({"query": "ranking"}))
assert [entry["uri"] for entry in result["results"]] == [
"viking://skills/positive",
"viking://memories/missing",
"viking://resources/negative",
]
assert [entry["score"] for entry in result["results"]] == [0.1, 0.0, -0.25]
assert result["total"] == 3
def test_tool_add_resource_uploads_existing_local_file(tmp_path):
sample = tmp_path / "sample.md"
sample.write_text("# Local resource\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.upload_temp_file.return_value = "upload_sample.md"
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/sample"},
}
result = json.loads(provider._tool_add_resource({
"url": str(sample),
"reason": "local test",
"wait": True,
}))
provider._client.upload_temp_file.assert_called_once_with(sample)
provider._client.post.assert_called_once_with("/api/v1/resources", {
"reason": "local test",
"wait": True,
"source_name": "sample.md",
"temp_file_id": "upload_sample.md",
})
assert result["status"] == "added"
assert result["root_uri"] == "viking://resources/sample"
def test_tool_add_resource_uploads_file_uri(tmp_path):
sample = tmp_path / "sample.md"
sample.write_text("# Local resource\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.upload_temp_file.return_value = "upload_sample.md"
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/sample"},
}
result = json.loads(provider._tool_add_resource({
"url": sample.as_uri(),
"reason": "file uri test",
}))
provider._client.upload_temp_file.assert_called_once_with(sample)
provider._client.post.assert_called_once_with("/api/v1/resources", {
"reason": "file uri test",
"source_name": "sample.md",
"temp_file_id": "upload_sample.md",
})
assert result["status"] == "added"
assert result["root_uri"] == "viking://resources/sample"
def test_tool_add_resource_uploads_existing_local_directory_and_cleans_zip(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
nested = docs / "nested"
nested.mkdir()
(nested / "api.md").write_text("# API\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
uploaded_paths = []
provider._client.upload_temp_file.side_effect = (
lambda path: uploaded_paths.append(path) or "upload_docs.zip"
)
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/docs"},
}
result = json.loads(provider._tool_add_resource({
"url": str(docs),
"reason": "directory test",
"wait": True,
}))
assert uploaded_paths
assert uploaded_paths[0].suffix == ".zip"
assert not uploaded_paths[0].exists()
provider._client.post.assert_called_once_with("/api/v1/resources", {
"reason": "directory test",
"wait": True,
"source_name": "docs",
"temp_file_id": "upload_docs.zip",
})
assert result["status"] == "added"
assert result["root_uri"] == "viking://resources/docs"
def test_tool_add_resource_directory_zip_skips_symlink_escape(tmp_path):
secret = tmp_path / "outside-secret.txt"
secret.write_text("do not upload\n", encoding="utf-8")
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
link = docs / "leak.txt"
try:
link.symlink_to(secret)
except OSError as exc:
pytest.skip(f"symlinks unavailable in test environment: {exc}")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
archive_entries = {}
def inspect_upload(path):
with zipfile.ZipFile(path) as archive:
archive_entries["names"] = archive.namelist()
archive_entries["payloads"] = {
name: archive.read(name)
for name in archive.namelist()
}
return "upload_docs.zip"
provider._client.upload_temp_file.side_effect = inspect_upload
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/docs"},
}
json.loads(provider._tool_add_resource({"url": str(docs)}))
assert archive_entries["names"] == ["guide.md"]
assert b"do not upload" not in b"".join(archive_entries["payloads"].values())
def test_tool_add_resource_cleans_local_directory_zip_when_add_fails(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
uploaded_paths = []
provider._client.upload_temp_file.side_effect = (
lambda path: uploaded_paths.append(path) or "upload_docs.zip"
)
provider._client.post.side_effect = RuntimeError("add failed")
with pytest.raises(RuntimeError, match="add failed"):
provider._tool_add_resource({"url": str(docs)})
assert uploaded_paths
assert not uploaded_paths[0].exists()
def test_tool_add_resource_cleans_local_directory_zip_when_upload_fails(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
uploaded_paths = []
def fail_upload(path):
uploaded_paths.append(path)
raise RuntimeError("upload failed")
provider._client.upload_temp_file.side_effect = fail_upload
with pytest.raises(RuntimeError, match="upload failed"):
provider._tool_add_resource({"url": str(docs)})
assert uploaded_paths
assert not uploaded_paths[0].exists()
provider._client.post.assert_not_called()
def test_tool_add_resource_rejects_missing_local_path(tmp_path):
missing = tmp_path / "missing.md"
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
result = json.loads(provider._tool_add_resource({"url": str(missing)}))
assert result["error"] == f"Local resource path does not exist: {missing}"
provider._client.upload_temp_file.assert_not_called()
provider._client.post.assert_not_called()
def test_tool_add_resource_sends_remote_url_as_path():
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/remote"},
}
provider._tool_add_resource({"url": "https://example.com/doc.md"})
provider._client.upload_temp_file.assert_not_called()
provider._client.post.assert_called_once_with("/api/v1/resources", {
"path": "https://example.com/doc.md",
})
@pytest.mark.parametrize("url", [
"git@github.com:org/repo.git",
"git@ssh.dev.azure.com:v3/org/project/repo",
"ssh://git@github.com/org/repo.git",
"git://github.com/org/repo.git",
])
def test_tool_add_resource_sends_git_remote_sources_as_path(url):
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/repo"},
}
provider._tool_add_resource({"url": url})
provider._client.upload_temp_file.assert_not_called()
provider._client.post.assert_called_once_with("/api/v1/resources", {
"path": url,
})
def test_viking_client_upload_temp_file_uses_multipart_identity_headers(tmp_path, monkeypatch):
sample = tmp_path / "sample.md"
sample.write_text("# Local resource\n", encoding="utf-8")
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="test-account",
user="test-user",
agent="test-agent",
)
captured_kwargs = {}
def capture_httpx_post(url, **kwargs):
captured_kwargs.update(kwargs)
return SimpleNamespace(
status_code=200,
text="",
json=lambda: {"status": "ok", "result": {"temp_file_id": "upload_sample.md"}},
raise_for_status=lambda: None,
)
monkeypatch.setattr(client._httpx, "post", capture_httpx_post)
assert client.upload_temp_file(sample) == "upload_sample.md"
assert "files" in captured_kwargs
assert "json" not in captured_kwargs
headers = captured_kwargs["headers"]
assert headers["X-OpenViking-Account"] == "test-account"
assert headers["X-OpenViking-User"] == "test-user"
assert headers["X-OpenViking-Agent"] == "test-agent"
assert headers["X-API-Key"] == "test-key"
assert "Content-Type" not in headers
def test_viking_client_raises_structured_server_error():
client = _VikingClient.__new__(_VikingClient)
response = SimpleNamespace(
status_code=403,
text='{"status":"error"}',
json=lambda: {
"status": "error",
"error": {
"code": "PERMISSION_DENIED",
"message": "direct host filesystem paths are not allowed",
},
},
raise_for_status=lambda: None,
)
with pytest.raises(RuntimeError, match="PERMISSION_DENIED"):
client._parse_response(response)
def test_viking_client_headers_include_bearer_when_api_key_set():
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="acct",
user="usr",
agent="hermes",
)
headers = client._headers()
assert headers["X-API-Key"] == "test-key"
assert headers["Authorization"] == "Bearer test-key"
def test_viking_client_headers_send_tenant_when_default():
# account/user set to the literal string "default". OpenViking 0.3.x
# requires X-OpenViking-Account and X-OpenViking-User for ROOT API key
# requests to tenant-scoped APIs — omitting them causes
# INVALID_ARGUMENT errors even when account="default".
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="default",
user="default",
agent="hermes",
)
headers = client._headers()
assert headers["X-OpenViking-Account"] == "default"
assert headers["X-OpenViking-User"] == "default"
assert headers["X-OpenViking-Agent"] == "hermes"
assert headers["Authorization"] == "Bearer test-key"
def test_viking_client_headers_send_tenant_when_empty_falls_back_to_default():
# Empty account/user strings fall back to "default" via the constructor.
# Headers are sent even for the default value — ROOT API keys need them.
client = _VikingClient(
"https://example.com",
api_key="",
account="",
user="",
agent="hermes",
)
headers = client._headers()
assert headers["X-OpenViking-Account"] == "default"
assert headers["X-OpenViking-User"] == "default"
assert "Authorization" not in headers
assert "X-API-Key" not in headers
def test_viking_client_headers_sent_with_real_tenant_values():
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="real-account",
user="real-user",
agent="hermes",
)
headers = client._headers()
assert headers["X-OpenViking-Account"] == "real-account"
assert headers["X-OpenViking-User"] == "real-user"
def test_viking_client_health_sends_auth_headers(monkeypatch):
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="",
user="",
agent="hermes",
)
captured = {}
def capture_get(url, **kwargs):
captured["url"] = url
captured["headers"] = kwargs.get("headers") or {}
return SimpleNamespace(status_code=200)
monkeypatch.setattr(client._httpx, "get", capture_get)
assert client.health() is True
assert captured["url"] == "https://example.com/health"
assert captured["headers"]["Authorization"] == "Bearer test-key"
tests/plugins/memory/test_supermemory_provider.py
+459
View File
@@ -0,0 +1,459 @@
import json
import os
import stat
import threading
import pytest
from plugins.memory.supermemory import (
SupermemoryMemoryProvider,
_clean_text_for_capture,
_format_prefetch_context,
_load_supermemory_config,
_save_supermemory_config,
)
class FakeClient:
def __init__(self, api_key: str, timeout: float, container_tag: str, search_mode: str = "hybrid"):
self.api_key = api_key
self.timeout = timeout
self.container_tag = container_tag
self.search_mode = search_mode
self.add_calls = []
self.search_results = []
self.profile_response = {"static": [], "dynamic": [], "search_results": []}
self.ingest_calls = []
self.forgotten_ids = []
self.forget_by_query_response = {"success": True, "message": "Forgot"}
def add_memory(self, content, metadata=None, *, entity_context="",
container_tag=None, custom_id=None):
self.add_calls.append({
"content": content,
"metadata": metadata,
"entity_context": entity_context,
"container_tag": container_tag,
"custom_id": custom_id,
})
return {"id": "mem_123"}
def search_memories(self, query, *, limit=5, container_tag=None, search_mode=None):
return self.search_results
def get_profile(self, query=None, *, container_tag=None):
return self.profile_response
def forget_memory(self, memory_id, *, container_tag=None):
self.forgotten_ids.append(memory_id)
def forget_by_query(self, query, *, container_tag=None):
return self.forget_by_query_response
def ingest_conversation(self, session_id, messages, metadata=None):
self.ingest_calls.append({"session_id": session_id, "messages": messages, "metadata": metadata})
@pytest.fixture
def provider(monkeypatch, tmp_path):
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
p = SupermemoryMemoryProvider()
p.initialize("session-1", hermes_home=str(tmp_path), platform="cli")
return p
def test_is_available_false_without_api_key(monkeypatch):
monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False)
p = SupermemoryMemoryProvider()
assert p.is_available() is False
def test_is_available_false_when_import_missing(monkeypatch):
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
import builtins
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
if name == "supermemory":
raise ImportError("missing")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", fake_import)
p = SupermemoryMemoryProvider()
assert p.is_available() is False
def test_load_and_save_config_round_trip(tmp_path):
_save_supermemory_config({"container_tag": "demo-tag", "auto_capture": False}, str(tmp_path))
cfg = _load_supermemory_config(str(tmp_path))
# container_tag is kept raw — sanitization happens in initialize() after template resolution
assert cfg["container_tag"] == "demo-tag"
assert cfg["auto_capture"] is False
assert cfg["auto_recall"] is True
def test_clean_text_for_capture_strips_injected_context():
text = "hello\n<supermemory-context>ignore me</supermemory-context>\nworld"
assert _clean_text_for_capture(text) == "hello\nworld"
def test_format_prefetch_context_deduplicates_overlap():
result = _format_prefetch_context(
static_facts=["Jordan prefers short answers"],
dynamic_facts=["Jordan prefers short answers", "Uses Hermes"],
search_results=[{"memory": "Uses Hermes", "similarity": 0.9}],
max_results=10,
)
assert result.count("Jordan prefers short answers") == 1
assert result.count("Uses Hermes") == 1
assert "<supermemory-context>" in result
def test_prefetch_includes_profile_on_first_turn(provider):
provider._client.profile_response = {
"static": ["Jordan prefers short answers"],
"dynamic": ["Current project is Supermemory provider"],
"search_results": [{"memory": "Working on Hermes memory provider", "similarity": 0.88}],
}
provider.on_turn_start(1, "start")
result = provider.prefetch("what am I working on?")
assert "User Profile (Persistent)" in result
assert "Recent Context" in result
assert "Relevant Memories" in result
def test_prefetch_skips_profile_between_frequency(provider):
provider._client.profile_response = {
"static": ["Jordan prefers short answers"],
"dynamic": ["Current project is Supermemory provider"],
"search_results": [{"memory": "Working on Hermes memory provider", "similarity": 0.88}],
}
provider.on_turn_start(2, "next")
result = provider.prefetch("what am I working on?")
assert "Relevant Memories" in result
assert "User Profile (Persistent)" not in result
def test_sync_turn_buffers_short_messages(provider):
# Trivial filtering is no longer applied at sync time — every non-empty turn
# is buffered and only the full session is written at session boundaries.
provider.sync_turn("ok", "sure", session_id="session-1")
assert provider._session_turns == [{"user": "ok", "assistant": "sure"}]
assert provider._client.add_calls == []
def test_sync_turn_buffers_cleaned_exchange(provider):
provider.sync_turn(
"Please remember this\n<supermemory-context>ignore</supermemory-context>",
"Got it, storing the context",
session_id="session-1",
)
assert len(provider._session_turns) == 1
turn = provider._session_turns[0]
assert "ignore" not in turn["user"]
assert turn["user"].startswith("Please remember this")
assert turn["assistant"] == "Got it, storing the context"
# Buffering only — no per-turn writes to the client
assert provider._client.add_calls == []
assert provider._client.ingest_calls == []
def test_on_session_end_ingests_clean_messages(provider):
messages = [
{"role": "system", "content": "skip"},
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi there"},
]
provider.on_session_end(messages)
assert len(provider._client.ingest_calls) == 1
payload = provider._client.ingest_calls[0]
assert payload["session_id"] == "session-1"
assert payload["messages"] == [
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi there"},
]
assert payload["metadata"]["type"] == "full_session"
assert payload["metadata"]["session_id"] == "session-1"
assert payload["metadata"]["message_count"] == 2
# Buffer is cleared after a normal session-end ingest.
assert provider._session_turns == []
def test_merge_metadata_stamps_sm_source():
# sm_source routes Hermes writes into the "Hermes" Space in the Supermemory
# app (functional routing, not telemetry) — must always be present.
from plugins.memory.supermemory import _SupermemoryClient
client = _SupermemoryClient.__new__(_SupermemoryClient)
merged = client._merge_metadata({"type": "explicit_memory"})
assert merged["sm_source"] == "hermes"
assert merged["type"] == "explicit_memory"
# Legacy "source" is migrated into "type" when type is absent.
merged2 = client._merge_metadata({"source": "conversation_turn"})
assert merged2["sm_source"] == "hermes"
assert merged2["type"] == "conversation_turn"
assert "source" not in merged2
def test_on_memory_write_tracks_thread(provider):
provider.on_memory_write("add", "memory", "Jordan likes concise docs")
assert provider._write_thread is not None
provider._write_thread.join(timeout=1)
assert len(provider._client.add_calls) == 1
assert provider._client.add_calls[0]["metadata"]["type"] == "explicit_memory"
def test_shutdown_joins_threads_and_flushes_buffer(provider, monkeypatch):
started = threading.Event()
release = threading.Event()
def slow_add_memory(content, metadata=None, *, entity_context="",
container_tag=None, custom_id=None):
started.set()
release.wait(timeout=1)
provider._client.add_calls.append({
"content": content,
"metadata": metadata,
"entity_context": entity_context,
})
return {"id": "mem_slow"}
monkeypatch.setattr(provider._client, "add_memory", slow_add_memory)
# sync_turn now only buffers — no thread is spawned.
provider.sync_turn(
"Please remember this request in long-term memory",
"Absolutely, I will keep that in long-term memory.",
session_id="session-1",
)
assert provider._sync_thread is None
assert len(provider._session_turns) == 1
# on_memory_write still runs on a background thread.
provider.on_memory_write("add", "memory", "Jordan likes concise docs")
assert started.wait(timeout=1)
assert provider._write_thread is not None
release.set()
provider.shutdown()
# All tracked threads joined and cleared.
assert provider._sync_thread is None
assert provider._write_thread is None
assert provider._prefetch_thread is None
# Explicit memory write went through.
assert len(provider._client.add_calls) == 1
# Buffered turn was flushed as a partial full-session ingest.
assert len(provider._client.ingest_calls) == 1
payload = provider._client.ingest_calls[0]
assert payload["session_id"] == "session-1"
assert payload["metadata"]["partial"] is True
assert payload["metadata"]["type"] == "full_session"
def test_store_tool_returns_saved_payload(provider):
result = json.loads(provider.handle_tool_call("supermemory_store", {"content": "Jordan likes concise docs"}))
assert result["saved"] is True
assert result["id"] == "mem_123"
def test_search_tool_formats_results(provider):
provider._client.search_results = [
{"id": "m1", "memory": "Jordan likes concise docs", "similarity": 0.92}
]
result = json.loads(provider.handle_tool_call("supermemory_search", {"query": "concise docs"}))
assert result["count"] == 1
assert result["results"][0]["similarity"] == 92
def test_forget_tool_by_id(provider):
result = json.loads(provider.handle_tool_call("supermemory_forget", {"id": "m1"}))
assert result == {"forgotten": True, "id": "m1"}
assert provider._client.forgotten_ids == ["m1"]
def test_forget_tool_by_query(provider):
provider._client.forget_by_query_response = {"success": True, "message": "Forgot one", "id": "m7"}
result = json.loads(provider.handle_tool_call("supermemory_forget", {"query": "that thing"}))
assert result["success"] is True
assert result["id"] == "m7"
def test_profile_tool_formats_sections(provider):
provider._client.profile_response = {
"static": ["Jordan prefers concise docs"],
"dynamic": ["Working on Supermemory provider"],
"search_results": [],
}
result = json.loads(provider.handle_tool_call("supermemory_profile", {}))
assert result["static_count"] == 1
assert result["dynamic_count"] == 1
assert "User Profile (Persistent)" in result["profile"]
def test_handle_tool_call_returns_error_when_unconfigured(monkeypatch):
monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False)
p = SupermemoryMemoryProvider()
result = json.loads(p.handle_tool_call("supermemory_search", {"query": "x"}))
assert "error" in result
# -- Identity template tests --------------------------------------------------
def test_identity_template_resolved_in_container_tag(monkeypatch, tmp_path):
"""container_tag with {identity} resolves to profile-scoped tag."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"container_tag": "hermes-{identity}"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli", agent_identity="coder")
assert p._container_tag == "hermes_coder"
def test_identity_template_default_profile(monkeypatch, tmp_path):
"""Without agent_identity kwarg, {identity} resolves to 'default'."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"container_tag": "hermes-{identity}"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._container_tag == "hermes_default"
def test_container_tag_env_var_override(monkeypatch, tmp_path):
"""SUPERMEMORY_CONTAINER_TAG env var overrides config."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setenv("SUPERMEMORY_CONTAINER_TAG", "env-override")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._container_tag == "env_override"
# -- Search mode tests --------------------------------------------------------
def test_search_mode_config_passed_to_client(monkeypatch, tmp_path):
"""search_mode from config is passed to _SupermemoryClient."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"search_mode": "memories"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._search_mode == "memories"
assert p._client.search_mode == "memories"
def test_invalid_search_mode_falls_back_to_default(monkeypatch, tmp_path):
"""Invalid search_mode falls back to 'hybrid'."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"search_mode": "invalid_mode"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._search_mode == "hybrid"
# -- Multi-container tests ----------------------------------------------------
def test_multi_container_disabled_by_default(provider):
"""Multi-container is off by default; schemas have no container_tag param."""
assert provider._enable_custom_containers is False
schemas = provider.get_tool_schemas()
for s in schemas:
assert "container_tag" not in s["parameters"]["properties"]
def test_multi_container_enabled_adds_schema_param(monkeypatch, tmp_path):
"""When enabled, tool schemas include container_tag parameter."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["project-alpha", "shared"],
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._enable_custom_containers is True
assert p._allowed_containers == ["hermes", "project_alpha", "shared"]
schemas = p.get_tool_schemas()
for s in schemas:
assert "container_tag" in s["parameters"]["properties"]
def test_multi_container_tool_store_with_custom_tag(monkeypatch, tmp_path):
"""supermemory_store uses the resolved container_tag when multi-container is enabled."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["project-alpha"],
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
result = json.loads(p.handle_tool_call("supermemory_store", {
"content": "test memory",
"container_tag": "project-alpha",
}))
assert result["saved"] is True
assert result["container_tag"] == "project_alpha"
assert p._client.add_calls[-1]["container_tag"] == "project_alpha"
def test_multi_container_rejects_unlisted_tag(monkeypatch, tmp_path):
"""Tool calls with a non-whitelisted container_tag return an error."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["allowed-tag"],
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
result = json.loads(p.handle_tool_call("supermemory_store", {
"content": "test",
"container_tag": "forbidden-tag",
}))
assert "error" in result
assert "not allowed" in result["error"]
def test_multi_container_system_prompt_includes_instructions(monkeypatch, tmp_path):
"""system_prompt_block includes container list and instructions when multi-container is enabled."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["docs"],
"custom_container_instructions": "Use docs for documentation context.",
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
block = p.system_prompt_block()
assert "Multi-container mode enabled" in block
assert "docs" in block
assert "Use docs for documentation context." in block
def test_get_config_schema_minimal():
"""get_config_schema only returns the API key field."""
p = SupermemoryMemoryProvider()
schema = p.get_config_schema()
assert len(schema) == 1
assert schema[0]["key"] == "api_key"
assert schema[0]["secret"] is True
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode bits not enforced on Windows")
def test_save_config_sets_owner_only_permissions(tmp_path):
"""supermemory.json must be written with 0o600 so API key is not world-readable."""
_save_supermemory_config({"api_key": "sm-test-key"}, str(tmp_path))
config_file = tmp_path / "supermemory.json"
assert config_file.exists()
mode = stat.S_IMODE(config_file.stat().st_mode)
assert mode == 0o600, f"Expected 0o600 (owner-only), got {oct(mode)}"
tests/plugins/model_providers/test_deepseek_profile.py
+207
View File
@@ -0,0 +1,207 @@
"""Unit tests for the DeepSeek provider profile's thinking-mode wiring.
DeepSeek V4 (and the legacy ``deepseek-reasoner``) expects every request to
carry an explicit ``extra_body.thinking`` parameter. Omitting it makes the
server default to thinking-mode ON, which then enforces the
``reasoning_content``-must-be-echoed-back contract on subsequent turns and
breaks the conversation with HTTP 400 (#15700, #17212, #17825).
These tests pin the profile's wire-shape contract so DeepSeek requests stay
correctly shaped without going live.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def deepseek_profile():
"""Resolve the registered DeepSeek profile.
Going through ``providers.get_provider_profile`` keeps the test honest —
if someone later replaces the registered class with a plain
``ProviderProfile``, every assertion below collapses.
"""
# ``model_tools`` triggers plugin discovery on import, which is what
# registers the DeepSeek profile in the global provider registry.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("deepseek")
assert profile is not None, "deepseek provider profile must be registered"
return profile
class TestDeepSeekThinkingWireShape:
"""``build_api_kwargs_extras`` produces DeepSeek's exact wire format."""
def test_v4_pro_default_enables_thinking_without_effort(self, deepseek_profile):
"""No reasoning_config → thinking enabled, server picks default effort."""
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config=None, model="deepseek-v4-pro"
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_v4_pro_enabled_with_high_effort(self, deepseek_profile):
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_standard_efforts_pass_through(self, deepseek_profile, effort):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek-v4-pro",
)
assert top_level == {"reasoning_effort": effort}
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
def test_xhigh_and_max_normalize_to_max(self, deepseek_profile, effort):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek-v4-pro",
)
assert top_level == {"reasoning_effort": "max"}
def test_explicitly_disabled_sends_disabled_marker(self, deepseek_profile):
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
The crucial bit is that the parameter is *sent* at all — DeepSeek
defaults to thinking-on when ``thinking`` is absent.
"""
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}, model="deepseek-v4-pro"
)
assert extra_body == {"thinking": {"type": "disabled"}}
# No effort when disabled — DeepSeek rejects it.
assert top_level == {}
def test_disabled_ignores_effort_field(self, deepseek_profile):
"""Effort silently dropped when thinking is off."""
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"},
model="deepseek-v4-pro",
)
assert top_level == {}
def test_unknown_effort_omits_top_level(self, deepseek_profile):
"""Garbage effort → omit reasoning_effort so DeepSeek applies its default."""
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "garbage"},
model="deepseek-v4-pro",
)
assert top_level == {}
def test_empty_effort_omits_top_level(self, deepseek_profile):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": ""},
model="deepseek-v4-pro",
)
assert top_level == {}
class TestDeepSeekModelGating:
"""V4 family + ``deepseek-reasoner`` get thinking; V3 stays untouched."""
@pytest.mark.parametrize(
"model",
[
"deepseek-v4-pro",
"deepseek-v4-flash",
"deepseek-v4-future-variant",
"deepseek-reasoner",
"DEEPSEEK-V4-PRO", # case-insensitive
],
)
def test_thinking_capable_models_emit_thinking(self, deepseek_profile, model):
extra_body, _ = deepseek_profile.build_api_kwargs_extras(
reasoning_config=None, model=model
)
assert extra_body == {"thinking": {"type": "enabled"}}
@pytest.mark.parametrize(
"model",
[
"deepseek-chat", # V3 alias
"deepseek-v3-0324", # explicit V3
"deepseek-v3.1", # V3 minor revisions
"", # bare/unknown
None, # missing
"deepseek-unknown", # unrecognized
],
)
def test_non_thinking_models_emit_nothing(self, deepseek_profile, model):
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"}, model=model
)
assert extra_body == {}
assert top_level == {}
class TestDeepSeekFullKwargsIntegration:
"""End-to-end: the transport's full kwargs match DeepSeek's live wire format.
The live test harness in ``tests/run_agent/test_deepseek_v4_thinking_live.py``
sends ``{"reasoning_effort": "high", "extra_body": {"thinking": {"type":
"enabled"}}}``. Confirm the transport produces that exact shape when wired
through the registered DeepSeek profile.
"""
def test_full_kwargs_match_live_wire_shape(self, deepseek_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=deepseek_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://api.deepseek.com/v1",
provider_name="deepseek",
)
assert kwargs["model"] == "deepseek-v4-pro"
assert kwargs["reasoning_effort"] == "high"
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
def test_v3_chat_full_kwargs_omit_thinking(self, deepseek_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-chat",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=deepseek_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://api.deepseek.com/v1",
provider_name="deepseek",
)
assert "reasoning_effort" not in kwargs
assert "extra_body" not in kwargs or "thinking" not in kwargs.get("extra_body", {})
class TestDeepSeekAuxModel:
"""DeepSeek aux model is set on the profile so users stop seeing the
bogus 'No auxiliary LLM provider configured' warning (#26924).
Pinned at the profile layer rather than the legacy
`_API_KEY_PROVIDER_AUX_MODELS_FALLBACK` dict — new providers are
expected to set `default_aux_model` on `ProviderProfile`, and the
fallback dict only exists for providers that predate the profiles
system.
"""
def test_profile_advertises_deepseek_chat(self, deepseek_profile):
assert deepseek_profile.default_aux_model == "deepseek-chat"
def test_consumer_api_returns_deepseek_chat(self):
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("deepseek") == "deepseek-chat"
def test_consumer_api_returns_non_empty(self):
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("deepseek") != ""
tests/plugins/model_providers/test_kimi_profile.py
+130
View File
@@ -0,0 +1,130 @@
"""Unit tests for the Kimi/Moonshot provider profile's reasoning wiring.
Moonshot's OpenAI-compat endpoint (``api.moonshot.ai/v1``) treats
``extra_body.thinking`` and a top-level ``reasoning_effort`` as mutually
exclusive. The profile must send at most one of them — never both — so a
request can't trip "cannot specify both 'thinking' and 'reasoning_effort'".
This mirrors the kimi-k2 handling already shipped for the opencode-go relay
(see ``tests/plugins/model_providers/test_opencode_go_profile.py``).
"""
from __future__ import annotations
import pytest
@pytest.fixture
def kimi_profile():
"""Resolve the registered Kimi profile via the provider registry.
Importing ``model_tools`` triggers plugin discovery, which registers the
Kimi profile. Going through ``get_provider_profile`` keeps the test honest:
if the registered class is ever swapped for a plain ``ProviderProfile`` the
assertions below collapse.
"""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("kimi-coding")
assert profile is not None, "kimi-coding provider profile must be registered"
return profile
class TestKimiReasoningWireShape:
"""``build_api_kwargs_extras`` never emits thinking + reasoning_effort together."""
def test_no_config_enables_thinking_without_effort(self, kimi_profile):
"""No reasoning_config → thinking on, server picks the depth.
Regression guard: this path previously also sent
``reasoning_effort="medium"``, pairing thinking + effort on every
default call.
"""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(reasoning_config=None)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_explicit_effort_sends_effort_only(self, kimi_profile, effort):
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort}
)
assert top_level == {"reasoning_effort": effort}
assert "thinking" not in extra_body
def test_enabled_without_effort_falls_back_to_thinking(self, kimi_profile):
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True}
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize("effort", ["", "garbage", "xhigh", "max"])
def test_unrecognized_effort_falls_back_to_thinking(self, kimi_profile, effort):
"""Unknown/strong efforts aren't in Moonshot's low|medium|high set, so
we drop to the thinking toggle rather than sending an invalid effort."""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort}
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_disabled_sends_thinking_disabled_only(self, kimi_profile):
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
def test_disabled_ignores_effort(self, kimi_profile):
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"}
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
@pytest.mark.parametrize(
"reasoning_config",
[
None,
{"enabled": True},
{"enabled": True, "effort": "high"},
{"enabled": True, "effort": "garbage"},
{"enabled": False},
{"enabled": False, "effort": "low"},
],
)
def test_never_emits_both(self, kimi_profile, reasoning_config):
"""The core invariant: thinking and reasoning_effort are never both set."""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config=reasoning_config
)
assert not ("thinking" in extra_body and "reasoning_effort" in top_level)
class TestKimiFullKwargsIntegration:
"""The transport's full kwargs carry at most one reasoning knob."""
def _build(self, kimi_profile, reasoning_config):
from agent.transports.chat_completions import ChatCompletionsTransport
return ChatCompletionsTransport().build_kwargs(
model="kimi-k2-turbo-preview",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=kimi_profile,
reasoning_config=reasoning_config,
base_url="https://api.moonshot.ai/v1",
provider_name="kimi-coding",
)
def test_explicit_effort_omits_thinking(self, kimi_profile):
kwargs = self._build(kimi_profile, {"enabled": True, "effort": "high"})
assert kwargs["reasoning_effort"] == "high"
assert "thinking" not in kwargs.get("extra_body", {})
def test_no_config_omits_effort(self, kimi_profile):
kwargs = self._build(kimi_profile, None)
assert "reasoning_effort" not in kwargs
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
tests/plugins/model_providers/test_minimax_profile.py
+120
View File
@@ -0,0 +1,120 @@
"""Unit tests for the MiniMax provider profile.
Three MiniMax provider profiles (`minimax` direct API, `minimax-cn` China direct
API, `minimax-oauth` browser OAuth) all advertise a `default_aux_model` on
their `ProviderProfile`. The previous M2.7 / M2.7-highspeed values were
stale relative to the current frontier model (M3, released 2026-06-01) and
inconsistent with the `_PROVIDER_MODELS["minimax"]` catalog top entry in
`hermes_cli/models.py`.
This file pins the new defaults so the choice is reviewable and any future
revert shows up in a failing test rather than silent behavior drift.
Refs:
- Issue #36196: M3 support request
- PR #36205 (closed unmerged): Csrayz's M3 + 1M context work
- PR #36212 (open): adds M3 to `_PROVIDER_MODELS["minimax"]` catalog
- PR #6082: M2.7-highspeed → M2.7 for aux model (half-price fix)
- Commit 773a0faca: same profile-layer fix pattern for `deepseek`
"""
from __future__ import annotations
import pytest
@pytest.fixture(params=["minimax", "minimax-cn", "minimax-oauth"])
def minimax_profile(request):
"""Resolve each registered MiniMax profile.
Going through ``providers.get_provider_profile`` keeps the test honest —
if someone later replaces the registered class with a plain
``ProviderProfile``, every assertion below collapses.
"""
import model_tools # noqa: F401 -- triggers plugin discovery
import providers
profile = providers.get_provider_profile(request.param)
assert profile is not None, f"{request.param} provider profile must be registered"
return profile, request.param
class TestMinimaxAuxModelM3:
"""MiniMax profile aux model is the new frontier M3, not the stale M2.7.
The catalog top entry is ``MiniMax-M3`` in
``hermes_cli.models._PROVIDER_MODELS['minimax']`` and the
user-facing ``model.default`` for a Token-Plan install is M3,
so pinning the aux default to the same model keeps the runtime
consistent (same auth, same billing pool, same rate limits, no
surprise 2x-cost highspeed variant). M3 was released 2026-06-01
— picking it as the aux default matches the forward-looking
catalog order rather than the pre-M3 era.
"""
@pytest.mark.parametrize(
"provider_id,expected",
[
("minimax", "MiniMax-M3"),
("minimax-cn", "MiniMax-M3"),
# minimax-oauth sticks with M2.7: the OAuth / Coding Plan
# tier historically used -highspeed (PR #6082 collapsed that
# to plain M2.7 to avoid the 2x TPS surcharge). M3 is not on
# the OAuth/Coding Plan tier per platform docs as of this PR,
# so the safe choice is the cheapest generally-available
# M2.7 — matching PR #6082's intent.
("minimax-oauth", "MiniMax-M2.7"),
],
)
def test_profile_advertises_expected_aux_model(
self, provider_id, expected
):
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile(provider_id)
assert profile is not None
assert profile.default_aux_model == expected, (
f"{provider_id} default_aux_model drifted to "
f"{profile.default_aux_model!r}, expected {expected!r}"
)
def test_consumer_api_returns_non_empty_for_each_provider(self, minimax_profile):
from agent.auxiliary_client import _get_aux_model_for_provider
profile, provider_id = minimax_profile
resolved = _get_aux_model_for_provider(provider_id)
assert resolved != "", (
f"_get_aux_model_for_provider({provider_id!r}) returned empty — "
"the 'No auxiliary LLM provider configured' warning will fire on "
f"every {provider_id} session even though the profile advertises "
f"default_aux_model={profile.default_aux_model!r}"
)
assert resolved == profile.default_aux_model, (
f"_get_aux_model_for_provider({provider_id!r}) returned "
f"{resolved!r} but profile advertises {profile.default_aux_model!r} "
"— the consumer API and the profile have drifted out of sync"
)
class TestMinimaxAuxModelNotHighspeed:
"""Regression guard against re-introducing the M2.7-highspeed aux default.
PR #6082 collapsed the highspeed aux choice to plain M2.7 because the
highspeed variant costs 2x with no real benefit for compression / vision /
session-search aux tasks. None of the three MiniMax profiles should
silently re-introduce that 2x-cost path.
"""
@pytest.mark.parametrize("provider_id", ["minimax", "minimax-cn", "minimax-oauth"])
def test_default_aux_model_is_not_highspeed(self, provider_id):
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile(provider_id)
assert profile is not None
assert "highspeed" not in profile.default_aux_model.lower(), (
f"{provider_id} default_aux_model={profile.default_aux_model!r} "
"is a -highspeed variant — that costs 2x for the same model and "
"broke #4082 the first time. Revert to plain M2.7 or M3."
)
tests/plugins/model_providers/test_opencode_go_profile.py
+180
View File
@@ -0,0 +1,180 @@
"""Unit tests for OpenCode Go reasoning-control wiring."""
from __future__ import annotations
import pytest
@pytest.fixture
def opencode_go_profile():
"""Resolve the registered OpenCode Go provider profile."""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("opencode-go")
assert profile is not None, "opencode-go provider profile must be registered"
return profile
class TestOpenCodeGoKimiReasoning:
"""Kimi K2 models use Moonshot's thinking + reasoning_effort shape on OpenCode Go."""
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="kimi-k2.6",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "high"}
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
model="kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
def test_minimal_effort_enables_thinking_without_effort(self, opencode_go_profile):
# "minimal" is not a Moonshot-supported value — drop it, keep thinking on.
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "minimal"},
model="kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize(
"effort",
[
"xhigh",
"max",
],
)
def test_strong_efforts_clamp_to_high(self, opencode_go_profile, effort):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="moonshotai/kimi-k2.6",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "high"}
def test_low_and_medium_pass_through(self, opencode_go_profile):
for effort in ("low", "medium"):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="kimi-k2.5",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": effort}
def test_no_config_preserves_server_default(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config=None,
model="kimi-k2.6",
)
assert extra_body == {}
assert top_level == {}
class TestOpenCodeGoDeepSeekThinking:
"""DeepSeek V4 models use DeepSeek-style thinking controls on OpenCode Go."""
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="deepseek-v4-pro",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "high"}
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
def test_no_config_emits_thinking_enabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config=None,
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_minimal_effort_enables_thinking_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "minimal"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_xhigh_and_max_normalize_to_max(self, opencode_go_profile):
for effort in ("xhigh", "max"):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek/deepseek-v4-pro",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "max"}
class TestOpenCodeGoModelGating:
"""Other OpenCode Go models must not receive Kimi/DeepSeek controls."""
@pytest.mark.parametrize(
"model",
[
"glm-5.1",
"qwen3.6-plus",
"minimax-m2.7",
"deepseek-v3.1",
"deepseek-chat",
"",
None,
],
)
def test_non_target_models_emit_nothing(self, opencode_go_profile, model):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model=model,
)
assert extra_body == {}
assert top_level == {}
class TestOpenCodeGoFullKwargsIntegration:
"""End-to-end transport kwargs include the profile-provided controls."""
def test_kimi_reasoning_reaches_extra_body_and_top_level(self, opencode_go_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="kimi-k2.6",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_go_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://opencode.ai/zen/go/v1",
)
assert "extra_body" not in kwargs
assert kwargs["reasoning_effort"] == "high"
def test_deepseek_thinking_reaches_extra_body_and_top_level(
self, opencode_go_profile
):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_go_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://opencode.ai/zen/go/v1",
)
assert "extra_body" not in kwargs
assert kwargs["reasoning_effort"] == "high"
tests/plugins/platforms/photon/test_auth.py
+609
View File
@@ -0,0 +1,609 @@
"""Tests for the Photon auth module (device login + dashboard API)."""
from __future__ import annotations
import json
import os
from base64 import b64encode
from pathlib import Path
from typing import Any, Dict
import pytest
from plugins.platforms.photon import auth as photon_auth
# ---------------------------------------------------------------------------
# Fake httpx — we don't want to hit the real Photon API in unit tests.
class _FakeResponse:
def __init__(
self,
*,
status: int = 200,
json_body: Any = None,
headers: Dict[str, str] | None = None,
text: str = "",
) -> None:
self.status_code = status
self._json = json_body if json_body is not None else {}
self.headers = headers or {}
self.text = text
def json(self) -> Any:
return self._json
def raise_for_status(self) -> None:
if self.status_code >= 400:
raise RuntimeError(f"HTTP {self.status_code}")
_PHOTON_ENV = (
"PHOTON_PROJECT_ID",
"PHOTON_PROJECT_SECRET",
"PHOTON_DASHBOARD_PROJECT_ID",
"PHOTON_SPECTRUM_HOST",
"PHOTON_ALLOWED_USERS",
"PHOTON_HOME_CHANNEL",
)
@pytest.fixture
def tmp_hermes_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
home = tmp_path / "hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
for key in _PHOTON_ENV:
monkeypatch.delenv(key, raising=False)
yield home
# save_env_value() mutates os.environ directly, so scrub any leakage.
for key in _PHOTON_ENV:
os.environ.pop(key, None)
# ---------------------------------------------------------------------------
# Credential storage
def test_store_and_load_photon_token(tmp_hermes_home: Path) -> None:
photon_auth.store_photon_token("abc123def456")
assert photon_auth.load_photon_token() == "abc123def456"
auth_json = json.loads((tmp_hermes_home / "auth.json").read_text())
assert auth_json["credential_pool"]["photon"][0]["access_token"] == "abc123def456"
def test_store_project_credentials_round_trip(
tmp_hermes_home: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
# Don't touch .env / os.environ here — exercise the auth.json path.
monkeypatch.setattr(photon_auth, "_persist_runtime_env", lambda *a, **k: None)
photon_auth.store_project_credentials(
spectrum_project_id="sp-123",
project_secret="secret-key",
dashboard_project_id="dash-456",
name="Hermes Agent",
)
for key in _PHOTON_ENV:
monkeypatch.delenv(key, raising=False)
sid, secret = photon_auth.load_project_credentials()
assert sid == "sp-123"
assert secret == "secret-key"
assert photon_auth.load_dashboard_project_id() == "dash-456"
def test_store_project_credentials_writes_env(tmp_hermes_home: Path) -> None:
photon_auth.store_project_credentials(
spectrum_project_id="sp-789",
project_secret="sek-ret",
dashboard_project_id="dash-1",
)
env_text = (tmp_hermes_home / ".env").read_text()
assert "PHOTON_PROJECT_ID=sp-789" in env_text
assert "PHOTON_PROJECT_SECRET=sek-ret" in env_text
def test_store_user_numbers_round_trip(tmp_hermes_home: Path) -> None:
photon_auth.store_user_numbers(
phone_number="+15551234567",
assigned_phone_number="+16282679185",
user_id="user-uuid",
dashboard_project_id="dash-uuid",
)
phone, assigned = photon_auth.load_user_numbers()
assert phone == "+15551234567"
assert assigned == "+16282679185"
summary = photon_auth.credential_summary()
assert summary["phone_number"] == "+15551234567"
assert summary["assigned_phone_number"] == "+16282679185"
rendered: list[str] = []
photon_auth.print_credential_summary(rendered.append)
assert " my number : +15551234567" in rendered[0]
assert " assigned number : +16282679185" in rendered[0]
def test_load_user_numbers_falls_back_to_home_channel(
tmp_hermes_home: Path,
) -> None:
from hermes_cli.config import save_env_value
save_env_value("PHOTON_HOME_CHANNEL", "+15551234567")
phone, assigned = photon_auth.load_user_numbers()
assert phone == "+15551234567"
assert assigned is None
def test_refresh_user_numbers_reads_existing_assignment(
tmp_hermes_home: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
photon_auth.store_user_numbers(phone_number="+15551234567")
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
assert kwargs.get("headers", {}).get("Authorization") == (
"Basic " + b64encode(b"sp:secret").decode("ascii")
)
assert url.endswith("/projects/sp/users/")
return _FakeResponse(json_body={"succeed": True, "data": {"users": [{
"id": "user-uuid",
"phoneNumber": "+1 (555) 123-4567",
"assignedPhoneNumber": "+16282679185",
}]}})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
phone, assigned = photon_auth.refresh_user_numbers("sp", "secret")
assert phone == "+15551234567"
assert assigned == "+16282679185"
assert photon_auth.load_user_numbers() == ("+15551234567", "+16282679185")
def test_load_project_credentials_env_override(
tmp_hermes_home: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(photon_auth, "_persist_runtime_env", lambda *a, **k: None)
photon_auth.store_project_credentials(
spectrum_project_id="from-file", project_secret="secret-file",
)
monkeypatch.setenv("PHOTON_PROJECT_ID", "from-env")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "secret-env")
sid, secret = photon_auth.load_project_credentials()
assert sid == "from-env"
assert secret == "secret-env"
# ---------------------------------------------------------------------------
# Device login flow
def test_request_device_code_uses_photon_cli(monkeypatch: pytest.MonkeyPatch) -> None:
captured: Dict[str, Any] = {}
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
captured["url"] = url
captured["body"] = kwargs.get("json")
return _FakeResponse(json_body={
"device_code": "dev-code-xyz",
"user_code": "ABCD-1234",
"verification_uri": "https://app.photon.codes/device",
"verification_uri_complete": "https://app.photon.codes/device?code=ABCD-1234",
"expires_in": 600,
"interval": 5,
})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
code = photon_auth.request_device_code()
assert code.device_code == "dev-code-xyz"
assert code.user_code == "ABCD-1234"
assert "/api/auth/device/code" in captured["url"]
# Hosted Photon allowlists registered device clients — an unregistered
# client_id is rejected with 400 invalid_client. We use Photon's published
# CLI device client and send the standard scope.
assert captured["body"]["client_id"] == "photon-cli"
assert captured["body"]["scope"] == "openid profile email"
def _device_code() -> "photon_auth.DeviceCode":
return photon_auth.DeviceCode(
device_code="d", user_code="u",
verification_uri="https://x", verification_uri_complete=None,
expires_in=10, interval=0,
)
def test_poll_for_token_body_access_token(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(status=200, json_body={"access_token": "tok-body"})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
assert photon_auth.poll_for_token(_device_code(), interval=0, timeout=2) == "tok-body"
def test_poll_for_token_session_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(status=200, json_body={"session": {"access_token": "tok-sess"}})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
assert photon_auth.poll_for_token(_device_code(), interval=0, timeout=2) == "tok-sess"
def test_poll_for_token_header_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(status=200, json_body={}, headers={"set-auth-token": "tok-hdr"})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
assert photon_auth.poll_for_token(_device_code(), interval=0, timeout=2) == "tok-hdr"
def test_poll_for_token_pending_then_success(monkeypatch: pytest.MonkeyPatch) -> None:
calls = {"n": 0}
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
calls["n"] += 1
if calls["n"] == 1:
return _FakeResponse(status=400, json_body={"error": "authorization_pending"})
return _FakeResponse(status=200, json_body={"access_token": "tok-eventual"})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
assert photon_auth.poll_for_token(_device_code(), interval=0, timeout=5) == "tok-eventual"
assert calls["n"] == 2
def test_poll_for_token_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(status=400, json_body={"error": "access_denied"})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
with pytest.raises(RuntimeError, match="access_denied"):
photon_auth.poll_for_token(_device_code(), interval=0, timeout=2)
# ---------------------------------------------------------------------------
# Projects
def test_list_projects_unwraps_list(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body=[{"id": "p1", "name": "Hermes Agent"}])
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
projects = photon_auth.list_projects("tok")
assert projects[0]["id"] == "p1"
def test_find_project_by_name_case_insensitive(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body={"data": [
{"id": "p1", "name": "Other"},
{"id": "p2", "name": "hermes agent"},
]})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
proj = photon_auth.find_project_by_name("tok", "Hermes Agent")
assert proj is not None and proj["id"] == "p2"
def test_create_project_sends_spectrum_true(monkeypatch: pytest.MonkeyPatch) -> None:
captured: Dict[str, Any] = {}
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
captured["url"] = url
captured["body"] = kwargs.get("json")
captured["headers"] = kwargs.get("headers")
return _FakeResponse(json_body={"success": True, "id": "new-proj"})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
data = photon_auth.create_project("tok", name="Hermes Agent")
assert data["id"] == "new-proj"
assert captured["body"]["spectrum"] is True
assert captured["body"]["name"] == "Hermes Agent"
assert captured["headers"]["Authorization"] == "Bearer tok"
assert captured["url"].endswith("/api/projects")
def test_create_project_raises_without_id(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body={"success": True})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
with pytest.raises(RuntimeError, match="project id"):
photon_auth.create_project("tok")
def test_ensure_spectrum_enabled_toggles_when_off(monkeypatch: pytest.MonkeyPatch) -> None:
get_calls = {"n": 0}
posted = {"toggle": False}
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
get_calls["n"] += 1
if get_calls["n"] == 1:
return _FakeResponse(json_body={"id": "p", "spectrum": False, "spectrumProjectId": None})
return _FakeResponse(json_body={"id": "p", "spectrum": True, "spectrumProjectId": "sp-1"})
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
if url.endswith("/spectrum/toggle"):
posted["toggle"] = True
return _FakeResponse(json_body={"success": True})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
proj = photon_auth.ensure_spectrum_enabled("tok", "p")
assert posted["toggle"] is True
assert proj["spectrumProjectId"] == "sp-1"
def test_ensure_spectrum_enabled_skips_toggle_when_on(monkeypatch: pytest.MonkeyPatch) -> None:
posted = {"toggle": False}
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body={"id": "p", "spectrum": True, "spectrumProjectId": "sp-1"})
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
if url.endswith("/spectrum/toggle"):
posted["toggle"] = True
return _FakeResponse(json_body={"success": True})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
proj = photon_auth.ensure_spectrum_enabled("tok", "p")
assert posted["toggle"] is False
assert proj["spectrumProjectId"] == "sp-1"
def test_regenerate_project_secret(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
assert url.endswith("/regenerate-secret")
return _FakeResponse(json_body={"success": True, "projectSecret": "rotated"})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
assert photon_auth.regenerate_project_secret("tok", "p") == "rotated"
# ---------------------------------------------------------------------------
# Users
def test_create_user_rejects_invalid_phone() -> None:
with pytest.raises(ValueError, match="E.164"):
photon_auth.create_user("proj", "secret", phone_number="not-a-number")
def test_create_user_posts_dashboard_shape(monkeypatch: pytest.MonkeyPatch) -> None:
captured: Dict[str, Any] = {}
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
captured["url"] = url
captured["body"] = kwargs.get("json")
captured["headers"] = kwargs.get("headers")
return _FakeResponse(json_body={"succeed": True, "data": {
"id": "user-uuid", "phoneNumber": "+15551234567",
}})
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
user = photon_auth.create_user("proj-id", "secret", phone_number="+15551234567")
assert user["id"] == "user-uuid"
assert captured["body"]["type"] == "shared"
assert captured["body"]["phoneNumber"] == "+15551234567"
assert captured["headers"]["Authorization"] == (
"Basic " + b64encode(b"proj-id:secret").decode("ascii")
)
assert captured["url"].endswith("/projects/proj-id/users/")
def test_register_user_if_absent_dedup(monkeypatch: pytest.MonkeyPatch) -> None:
posted = {"n": 0}
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body={"succeed": True, "data": {"users": [{
"id": "u1",
"phoneNumber": "+1 (555) 123-4567",
"assignedPhoneNumber": "+16282679185",
}]}})
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
posted["n"] += 1
return _FakeResponse(json_body={"success": True, "user": {}})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
# Same number, different formatting — should match and NOT create.
user, created = photon_auth.register_user_if_absent(
"proj", "secret", phone_number="+15551234567",
)
assert created is False
assert user["id"] == "u1"
assert posted["n"] == 0
# The reused user carries the assigned iMessage line ("TEXTS ON").
assert photon_auth.user_assigned_line(user) == "+16282679185"
def test_user_assigned_line() -> None:
assert (
photon_auth.user_assigned_line({"assignedPhoneNumber": "+16282679185"})
== "+16282679185"
)
# Own number present but no assignment yet (e.g. freshly created user).
assert photon_auth.user_assigned_line({"phoneNumber": "+15551234567"}) is None
assert photon_auth.user_assigned_line({"assignedPhoneNumber": ""}) is None
assert photon_auth.user_assigned_line({}) is None
assert photon_auth.user_assigned_line(None) is None
def test_register_user_if_absent_creates(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body={"succeed": True, "data": {"users": []}})
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body={"succeed": True, "data": {"id": "u-new"}})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
user, created = photon_auth.register_user_if_absent(
"proj", "secret", phone_number="+15551234567",
)
assert created is True
assert user["id"] == "u-new"
# ---------------------------------------------------------------------------
# Lines (assigned number)
def test_get_imessage_line_returns_existing(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body=[
{"id": "l1", "platform": "imessage", "phoneNumber": "+15559999999", "status": "active"},
])
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
line = photon_auth.get_imessage_line("tok", "proj")
assert line is not None and line["phoneNumber"] == "+15559999999"
def test_get_imessage_line_provisions_when_missing(monkeypatch: pytest.MonkeyPatch) -> None:
added = {"n": 0}
def fake_get(url: str, **kwargs: Any) -> _FakeResponse:
return _FakeResponse(json_body=[])
def fake_post(url: str, **kwargs: Any) -> _FakeResponse:
added["n"] += 1
assert kwargs.get("json", {}).get("platform") == "imessage"
return _FakeResponse(json_body={"success": True, "line": {
"id": "l-new", "platform": "imessage", "phoneNumber": "+15558888888",
}})
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
line = photon_auth.get_imessage_line("tok", "proj")
assert added["n"] == 1
assert line["phoneNumber"] == "+15558888888"
# ---------------------------------------------------------------------------
# Credential summary (no secret leakage)
def test_credential_summary_no_secret_leak(
tmp_hermes_home: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(photon_auth, "_persist_runtime_env", lambda *a, **k: None)
photon_auth.store_photon_token("token-aaaaaaaaaaaaaaaa")
photon_auth.store_project_credentials(
spectrum_project_id="sp-uuid",
project_secret="secret-bbbbbbbbbbb",
dashboard_project_id="dash-uuid",
)
summary = photon_auth.credential_summary()
blob = "\n".join(summary.values())
assert "token-aaaa" not in blob
assert "secret-bbbb" not in blob
assert summary["device_token"].startswith("✓")
assert summary["project_key"].startswith("✓")
assert summary["spectrum_project_id"] == "sp-uuid"
assert summary["dashboard_project_id"] == "dash-uuid"
assert summary["phone_number"].startswith("✗ missing")
assert summary["assigned_phone_number"].startswith("✗ missing")
# ---------------------------------------------------------------------------
# Device-token candidate extraction + dashboard validation.
def test_device_response_candidates_covers_known_shapes() -> None:
candidates = photon_auth._device_response_token_candidates(
{
"access_token": "tok-snake",
"accessToken": "tok-camel",
"data": {"access_token": "tok-data"},
},
headers={"set-auth-token": "Bearer tok-header"},
)
by_source = {c.source: c.token for c in candidates}
assert by_source["access_token"] == "tok-snake"
assert by_source["accessToken"] == "tok-camel"
assert by_source["data.access_token"] == "tok-data"
# "Bearer " prefix is stripped from the header value.
assert by_source["set-auth-token"] == "tok-header"
def test_device_response_candidates_dedupes() -> None:
candidates = photon_auth._device_response_token_candidates(
{"access_token": "same", "accessToken": "same"},
)
assert [c.token for c in candidates] == ["same"]
def test_validate_photon_token_rejects_unrecognized_session(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_get(url: str, *, headers: Dict[str, str], timeout: float) -> _FakeResponse:
if url.endswith("/api/auth/get-session"):
return _FakeResponse(json_body={}) # no "user" key
return _FakeResponse(json_body=[])
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
with pytest.raises(photon_auth.PhotonDashboardAuthError):
photon_auth.validate_photon_token("some-token")
def test_validate_photon_token_rejects_project_api_denial(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_get(url: str, *, headers: Dict[str, str], timeout: float) -> _FakeResponse:
if url.endswith("/api/auth/get-session"):
return _FakeResponse(json_body={"user": {"id": "u1"}})
return _FakeResponse(status=403) # project API rejects
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
with pytest.raises(photon_auth.PhotonDashboardAuthError):
photon_auth.validate_photon_token("some-token")
def test_login_device_flow_validates_before_persisting(
tmp_hermes_home: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_post(url: str, *, json: Dict[str, Any], timeout: float) -> _FakeResponse:
if url.endswith("/api/auth/device/code"):
return _FakeResponse(json_body={
"device_code": "dev", "user_code": "AAAA",
"verification_uri": "https://app.photon.codes/device",
"verification_uri_complete": None,
"expires_in": 600, "interval": 0,
})
# device/token approval
return _FakeResponse(json_body={"access_token": "good-token"})
def fake_get(url: str, *, headers: Dict[str, str], timeout: float) -> _FakeResponse:
if url.endswith("/api/auth/get-session"):
return _FakeResponse(json_body={"user": {"id": "u1"}})
return _FakeResponse(json_body=[]) # projects OK
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
token = photon_auth.login_device_flow(open_browser=False)
assert token == "good-token"
assert photon_auth.load_photon_token() == "good-token"
def test_login_device_flow_raises_when_token_invalid(
tmp_hermes_home: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_post(url: str, *, json: Dict[str, Any], timeout: float) -> _FakeResponse:
if url.endswith("/api/auth/device/code"):
return _FakeResponse(json_body={
"device_code": "dev", "user_code": "AAAA",
"verification_uri": "https://app.photon.codes/device",
"verification_uri_complete": None,
"expires_in": 600, "interval": 0,
})
return _FakeResponse(json_body={"access_token": "bad-token"})
def fake_get(url: str, *, headers: Dict[str, str], timeout: float) -> _FakeResponse:
return _FakeResponse(status=401) # session lookup rejects
monkeypatch.setattr(photon_auth.httpx, "post", fake_post)
monkeypatch.setattr(photon_auth.httpx, "get", fake_get)
with pytest.raises(photon_auth.PhotonDashboardAuthError):
photon_auth.login_device_flow(open_browser=False)
# A token that failed validation must never be persisted.
assert photon_auth.load_photon_token() is None
tests/plugins/platforms/photon/test_inbound.py
+314
View File
@@ -0,0 +1,314 @@
"""Inbound dispatch + dedup tests for PhotonAdapter.
These bypass the loopback HTTP stream — they call ``_dispatch_inbound`` /
``_on_inbound_line`` / ``_is_duplicate`` directly, exercising the
sidecar-event parsing without spawning the Node sidecar or binding ports.
"""
from __future__ import annotations
import base64
import json
from pathlib import Path
from typing import Any, Dict, List
import pytest
from gateway.config import Platform, PlatformConfig
from gateway.platforms.base import MessageEvent, MessageType
from plugins.platforms.photon.adapter import PhotonAdapter
def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
cfg = PlatformConfig(enabled=True, token="", extra={})
return PhotonAdapter(cfg)
def _capture(adapter: PhotonAdapter, monkeypatch: pytest.MonkeyPatch) -> List[MessageEvent]:
captured: List[MessageEvent] = []
async def fake_handle(event: MessageEvent) -> None:
captured.append(event)
monkeypatch.setattr(adapter, "handle_message", fake_handle)
return captured
def _dm_event(text: str, msg_id: str = "spc-msg-abc") -> Dict[str, Any]:
return {
"messageId": msg_id,
"platform": "iMessage",
"space": {"id": "+15551234567", "type": "dm", "phone": "+15551234567"},
"sender": {"id": "+15551234567"},
"content": {"type": "text", "text": text},
"timestamp": "2026-05-14T19:06:32.000Z",
}
@pytest.mark.asyncio
async def test_dispatch_text_dm(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
await adapter._dispatch_inbound(_dm_event("hello world"))
assert len(captured) == 1
event = captured[0]
assert event.text == "hello world"
assert event.message_type == MessageType.TEXT
assert event.message_id == "spc-msg-abc"
src = event.source
assert src is not None
assert src.platform == Platform("photon")
assert src.chat_id == "+15551234567"
assert src.chat_type == "dm"
assert src.user_id == "+15551234567"
@pytest.mark.asyncio
async def test_dispatch_group_type(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
event = {
"messageId": "spc-msg-grp",
"space": {"id": "group-guid-xyz", "type": "group", "phone": None},
"sender": {"id": "+15551234567"},
"content": {"type": "text", "text": "hi group"},
"timestamp": "2026-05-14T19:06:32.000Z",
}
await adapter._dispatch_inbound(event)
assert captured[0].source.chat_type == "group"
# A real 1x1 transparent PNG (passes base.py's _looks_like_image magic check).
_PNG_1X1_B64 = (
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhf"
"DwAChwGA60e6kgAAAABJRU5ErkJggg=="
)
def _attachment_event(
content: Dict[str, Any], msg_id: str = "spc-msg-att"
) -> Dict[str, Any]:
return {
"messageId": msg_id,
"space": {"id": "+15551234567", "type": "dm", "phone": "+15551234567"},
"sender": {"id": "+15551234567"},
"content": {"type": "attachment", **content},
"timestamp": "2026-05-14T19:06:32.000Z",
}
def _voice_event(
content: Dict[str, Any], msg_id: str = "spc-msg-voice"
) -> Dict[str, Any]:
return {
"messageId": msg_id,
"space": {"id": "+15551234567", "type": "dm", "phone": "+15551234567"},
"sender": {"id": "+15551234567"},
"content": {"type": "voice", **content},
"timestamp": "2026-05-14T19:06:32.000Z",
}
@pytest.mark.asyncio
async def test_dispatch_attachment_without_bytes_surfaces_marker(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""No inline ``data`` (over cap / failed sidecar read) -> text marker, no media."""
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
event = _attachment_event(
{"name": "IMG_4127.HEIC", "mimeType": "image/heic", "size": 12345}
)
await adapter._dispatch_inbound(event)
assert len(captured) == 1
ev = captured[0]
assert "Photon attachment received" in ev.text
assert "IMG_4127.HEIC" in ev.text
assert ev.message_type == MessageType.PHOTO
assert ev.media_urls == []
assert ev.media_types == []
@pytest.mark.asyncio
async def test_dispatch_attachment_downloads_image(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Inline base64 image bytes are decoded, cached, and exposed as media."""
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
raw = base64.b64decode(_PNG_1X1_B64)
event = _attachment_event(
{
"name": "photo.png",
"mimeType": "image/png",
"size": len(raw),
"data": _PNG_1X1_B64,
"encoding": "base64",
}
)
await adapter._dispatch_inbound(event)
assert len(captured) == 1
ev = captured[0]
assert ev.message_type == MessageType.PHOTO
assert ev.media_types == ["image/png"]
assert len(ev.media_urls) == 1
cached = Path(ev.media_urls[0])
try:
assert cached.is_file()
assert cached.read_bytes() == raw
assert ev.text == "(attachment)"
finally:
cached.unlink(missing_ok=True)
@pytest.mark.asyncio
async def test_dispatch_voice_downloads_audio(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Inbound Spectrum voice content is cached and routed to auto-STT."""
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
raw = b"OggS" + b"\x00" * 32
event = _voice_event(
{
"name": "note.ogg",
"mimeType": "audio/ogg",
"duration": 7,
"size": len(raw),
"data": base64.b64encode(raw).decode("ascii"),
"encoding": "base64",
}
)
await adapter._dispatch_inbound(event)
assert len(captured) == 1
ev = captured[0]
assert ev.message_type == MessageType.VOICE
assert ev.media_types == ["audio/ogg"]
assert len(ev.media_urls) == 1
cached = Path(ev.media_urls[0])
try:
assert cached.is_file()
assert cached.read_bytes() == raw
assert ev.text == "(voice)"
finally:
cached.unlink(missing_ok=True)
@pytest.mark.asyncio
async def test_dispatch_voice_without_bytes_surfaces_marker(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Metadata-only voice still tells the agent a voice note arrived."""
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
event = _voice_event(
{"name": "note.m4a", "mimeType": "audio/mp4", "duration": 12, "size": 12345}
)
await adapter._dispatch_inbound(event)
assert len(captured) == 1
ev = captured[0]
assert "Photon voice received" in ev.text
assert "note.m4a" in ev.text
assert "duration: 12s" in ev.text
assert ev.message_type == MessageType.VOICE
assert ev.media_urls == []
assert ev.media_types == []
@pytest.mark.asyncio
async def test_dispatch_attachment_downloads_document(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Non-image attachments route through the document cache as DOCUMENT."""
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
raw = b"%PDF-1.4 hermes test document"
event = _attachment_event(
{
"name": "report.pdf",
"mimeType": "application/pdf",
"size": len(raw),
"data": base64.b64encode(raw).decode("ascii"),
"encoding": "base64",
}
)
await adapter._dispatch_inbound(event)
assert len(captured) == 1
ev = captured[0]
assert ev.message_type == MessageType.DOCUMENT
assert ev.media_types == ["application/pdf"]
assert len(ev.media_urls) == 1
cached = Path(ev.media_urls[0])
try:
assert cached.is_file()
assert cached.read_bytes() == raw
assert ev.text == "(attachment)"
finally:
cached.unlink(missing_ok=True)
@pytest.mark.asyncio
async def test_on_inbound_line_dispatches_and_dedups(
monkeypatch: pytest.MonkeyPatch,
) -> None:
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
line = json.dumps(_dm_event("ping", msg_id="dup-1"))
await adapter._on_inbound_line(line)
await adapter._on_inbound_line(line) # same messageId -> deduped
assert len(captured) == 1
assert captured[0].text == "ping"
@pytest.mark.asyncio
async def test_on_inbound_line_ignores_bad_json(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
await adapter._on_inbound_line("{not json")
assert captured == []
def test_is_duplicate_window(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
assert adapter._is_duplicate("id-1") is False
assert adapter._is_duplicate("id-1") is True
assert adapter._is_duplicate("id-2") is False
assert adapter._is_duplicate("id-1") is True # still dup
def test_is_duplicate_hard_size_bound(monkeypatch: pytest.MonkeyPatch) -> None:
# A burst of unique ids within the window must not grow the dedup map past
# its bound — evict oldest (LRU), not only expired entries.
import plugins.platforms.photon.adapter as ad
monkeypatch.setattr(ad, "_DEDUP_MAX_SIZE", 5)
adapter = _make_adapter(monkeypatch)
for i in range(100):
adapter._is_duplicate(f"id-{i}")
assert len(adapter._seen_messages) <= 5
assert adapter._is_duplicate("id-99") is True # recent still deduped
assert adapter._is_duplicate("id-0") is False # oldest evicted
def test_check_requirements_without_node(monkeypatch: pytest.MonkeyPatch) -> None:
# If no node binary on PATH the adapter should refuse to start.
from plugins.platforms.photon import adapter as adapter_mod
monkeypatch.setattr(adapter_mod.shutil, "which", lambda _name: None)
assert adapter_mod.check_requirements() is False
tests/plugins/platforms/photon/test_markdown.py
+129
View File
@@ -0,0 +1,129 @@
"""Markdown handling tests for PhotonAdapter.
Markdown is on by default (the sidecar sends it via spectrum-ts'
``markdown()`` builder and iMessage renders it); ``PHOTON_MARKDOWN=false``
reverts to the stripped-plain-text path.
"""
from __future__ import annotations
from typing import Any, Dict, List, Tuple
import pytest
from gateway.config import PlatformConfig
from plugins.platforms.photon import adapter as photon_adapter
from plugins.platforms.photon.adapter import PhotonAdapter
_MD = "**bold** and `code`"
def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
cfg = PlatformConfig(enabled=True, token="", extra={})
return PhotonAdapter(cfg)
def _capture_sidecar(adapter: PhotonAdapter) -> List[Tuple[str, Dict[str, Any]]]:
calls: List[Tuple[str, Dict[str, Any]]] = []
async def _fake_call(path: str, body: Dict[str, Any]) -> Dict[str, Any]:
calls.append((path, body))
return {"ok": True, "messageId": "msg-123"}
adapter._sidecar_call = _fake_call # type: ignore[assignment]
return calls
def test_format_message_passthrough_by_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
adapter = _make_adapter(monkeypatch)
assert adapter.format_message(_MD) == _MD
def test_format_message_strips_when_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("PHOTON_MARKDOWN", "false")
adapter = _make_adapter(monkeypatch)
assert adapter.format_message(_MD) == "bold and code"
def test_supports_code_blocks_mirrors_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
assert _make_adapter(monkeypatch).supports_code_blocks is True
monkeypatch.setenv("PHOTON_MARKDOWN", "false")
assert _make_adapter(monkeypatch).supports_code_blocks is False
@pytest.mark.asyncio
async def test_sidecar_send_includes_markdown_format(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.send("+15551234567", _MD)
path, body = calls[0]
assert path == "/send"
assert body["format"] == "markdown"
assert body["text"] == _MD # passed through unstripped
@pytest.mark.asyncio
async def test_sidecar_send_omits_format_when_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Old-sidecar compat: the key is absent, not "text", when disabled."""
monkeypatch.setenv("PHOTON_MARKDOWN", "false")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.send("+15551234567", _MD)
_, body = calls[0]
assert "format" not in body
assert body["text"] == "bold and code"
@pytest.mark.asyncio
async def test_standalone_send_includes_markdown_format(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok")
posted: List[Tuple[str, Dict[str, Any]]] = []
class _Resp:
status_code = 200
@staticmethod
def json() -> Dict[str, Any]:
return {"ok": True, "messageId": "m-9"}
class _FakeClient:
def __init__(self, *a, **k):
pass
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
async def post(self, url: str, json: Dict[str, Any], headers=None):
posted.append((url, json))
return _Resp()
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient)
cfg = PlatformConfig(enabled=True, token="", extra={})
result = await photon_adapter._standalone_send(cfg, "+15551234567", _MD)
assert result.get("success") is True
assert posted[0][1]["format"] == "markdown"
tests/plugins/platforms/photon/test_mention_gating.py
+138
View File
@@ -0,0 +1,138 @@
"""Group-chat mention-gating tests for PhotonAdapter.
Parity with the BlueBubbles iMessage channel: when ``require_mention`` is
enabled, group messages are dropped unless they hit a wake-word pattern,
and the leading wake word is stripped from the ones that pass. DMs are
never gated.
These call ``_dispatch_inbound`` directly (no aiohttp / ports) and assert
on what reaches ``handle_message``.
"""
from __future__ import annotations
from typing import List
import pytest
from gateway.config import PlatformConfig
from gateway.platforms.base import MessageEvent
from plugins.platforms.photon.adapter import PhotonAdapter
def _make_adapter(monkeypatch: pytest.MonkeyPatch, extra: dict | None = None) -> PhotonAdapter:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
monkeypatch.delenv("PHOTON_REQUIRE_MENTION", raising=False)
monkeypatch.delenv("PHOTON_MENTION_PATTERNS", raising=False)
cfg = PlatformConfig(enabled=True, token="", extra=extra or {})
return PhotonAdapter(cfg)
def _group_payload(text: str) -> dict:
return {
"messageId": f"grp-{abs(hash(text))}",
"space": {"id": "group-guid-xyz", "type": "group", "phone": None},
"sender": {"id": "+15551234567"},
"content": {"type": "text", "text": text},
"timestamp": "2026-05-14T19:06:32.000Z",
}
def _dm_payload(text: str) -> dict:
return {
"messageId": f"dm-{abs(hash(text))}",
"space": {"id": "+15551234567", "type": "dm", "phone": "+15551234567"},
"sender": {"id": "+15551234567"},
"content": {"type": "text", "text": text},
"timestamp": "2026-05-14T19:06:32.000Z",
}
def _capture(adapter: PhotonAdapter, monkeypatch: pytest.MonkeyPatch) -> List[MessageEvent]:
captured: List[MessageEvent] = []
async def fake_handle(event: MessageEvent) -> None:
captured.append(event)
monkeypatch.setattr(adapter, "handle_message", fake_handle)
return captured
def test_require_mention_defaults_off(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
assert adapter.require_mention is False
# Defaults compile to the two Hermes wake-word patterns.
assert len(adapter._mention_patterns) == 2
@pytest.mark.asyncio
async def test_group_message_dropped_without_mention(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch, extra={"require_mention": True})
captured = _capture(adapter, monkeypatch)
await adapter._dispatch_inbound(_group_payload("just chatting, no wake word"))
assert captured == []
@pytest.mark.asyncio
async def test_group_message_passes_and_strips_wake_word(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch, extra={"require_mention": True})
captured = _capture(adapter, monkeypatch)
await adapter._dispatch_inbound(_group_payload("Hermes what's the weather"))
assert len(captured) == 1
# Leading wake word stripped before dispatch.
assert captured[0].text == "what's the weather"
@pytest.mark.asyncio
async def test_dm_never_gated(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch, extra={"require_mention": True})
captured = _capture(adapter, monkeypatch)
await adapter._dispatch_inbound(_dm_payload("no wake word here"))
assert len(captured) == 1
assert captured[0].text == "no wake word here"
@pytest.mark.asyncio
async def test_require_mention_off_passes_group_messages(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch) # require_mention defaults off
captured = _capture(adapter, monkeypatch)
await adapter._dispatch_inbound(_group_payload("plain group chatter"))
assert len(captured) == 1
assert captured[0].text == "plain group chatter"
def test_custom_mention_patterns_from_config(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(
monkeypatch,
extra={"require_mention": True, "mention_patterns": [r"(?<![\w@])@?amos\b[,:\-]?"]},
)
assert adapter.require_mention is True
assert len(adapter._mention_patterns) == 1
assert adapter._message_matches_mention_patterns("amos help me") is True
assert adapter._message_matches_mention_patterns("hermes help me") is False
def test_mention_patterns_env_comma_separated(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
monkeypatch.setenv("PHOTON_REQUIRE_MENTION", "true")
monkeypatch.setenv("PHOTON_MENTION_PATTERNS", r"bot\b, assistant\b")
cfg = PlatformConfig(enabled=True, token="", extra={})
adapter = PhotonAdapter(cfg)
assert adapter.require_mention is True
assert len(adapter._mention_patterns) == 2
assert adapter._message_matches_mention_patterns("hey bot") is True
def test_invalid_pattern_skipped(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(
monkeypatch,
extra={"require_mention": True, "mention_patterns": ["(unclosed", r"good\b"]},
)
# Bad regex dropped, good one kept.
assert len(adapter._mention_patterns) == 1
assert adapter._message_matches_mention_patterns("a good thing") is True
tests/plugins/platforms/photon/test_outbound_media.py
+255
View File
@@ -0,0 +1,255 @@
"""Outbound-media tests for PhotonAdapter.
Photon ships outbound attachments via spectrum-ts' ``attachment()`` /
``voice()`` content builders, reached through the Node sidecar's
``/send-attachment`` endpoint. These tests stub ``_sidecar_call`` so we
can assert the endpoint + body shape each ``send_*`` override produces
without spawning Node or binding ports.
"""
from __future__ import annotations
import os
from typing import Any, Dict, List, Tuple
import pytest
from gateway.config import PlatformConfig
from plugins.platforms.photon import adapter as photon_adapter
from plugins.platforms.photon.adapter import PhotonAdapter
def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
monkeypatch.delenv("PHOTON_WEBHOOK_SECRET", raising=False)
cfg = PlatformConfig(enabled=True, token="", extra={})
return PhotonAdapter(cfg)
def _capture_sidecar(adapter: PhotonAdapter) -> List[Tuple[str, Dict[str, Any]]]:
"""Replace ``_sidecar_call`` with a recorder that returns a fixed id."""
calls: List[Tuple[str, Dict[str, Any]]] = []
async def _fake_call(path: str, body: Dict[str, Any]) -> Dict[str, Any]:
calls.append((path, body))
return {"ok": True, "messageId": "msg-123"}
adapter._sidecar_call = _fake_call # type: ignore[assignment]
return calls
@pytest.fixture()
def real_file(tmp_path) -> str:
p = tmp_path / "photo.jpg"
p.write_bytes(b"\xff\xd8\xff\xe0fake-jpeg")
return str(p)
def _patch_safe_path(monkeypatch: pytest.MonkeyPatch) -> None:
"""Make path validation a passthrough so tmp files outside the cache pass."""
monkeypatch.setattr(
PhotonAdapter,
"validate_media_delivery_path",
staticmethod(lambda p: p if os.path.exists(p) else None),
)
@pytest.mark.asyncio
async def test_send_image_file_hits_attachment_endpoint(
monkeypatch: pytest.MonkeyPatch, real_file: str
) -> None:
_patch_safe_path(monkeypatch)
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
result = await adapter.send_image_file(
"any;-;+15551234567", real_file, caption="look"
)
assert result.success is True
assert result.message_id == "msg-123"
assert len(calls) == 1
path, body = calls[0]
assert path == "/send-attachment"
assert body["spaceId"] == "any;-;+15551234567"
assert body["path"] == real_file
assert body["kind"] == "attachment"
assert body["caption"] == "look"
assert body["mimeType"] == "image/jpeg" # inferred from .jpg
@pytest.mark.asyncio
async def test_send_voice_marks_kind_voice(
monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
_patch_safe_path(monkeypatch)
audio = tmp_path / "note.m4a"
audio.write_bytes(b"fake-audio")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
result = await adapter.send_voice("any;-;+1", str(audio))
assert result.success is True
path, body = calls[0]
assert path == "/send-attachment"
assert body["kind"] == "voice"
@pytest.mark.asyncio
async def test_send_document_passes_filename(
monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
_patch_safe_path(monkeypatch)
doc = tmp_path / "report.pdf"
doc.write_bytes(b"%PDF-1.4 fake")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.send_document("any;-;+1", str(doc), file_name="Q3.pdf")
_, body = calls[0]
assert body["kind"] == "attachment"
assert body["name"] == "Q3.pdf"
assert body["mimeType"] == "application/pdf"
@pytest.mark.asyncio
async def test_send_video_passes_through(
monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
_patch_safe_path(monkeypatch)
vid = tmp_path / "clip.mp4"
vid.write_bytes(b"fake-mp4")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.send_video("any;+;groupguid", str(vid), caption="watch")
_, body = calls[0]
assert body["kind"] == "attachment"
assert body["caption"] == "watch"
@pytest.mark.asyncio
async def test_send_image_url_caches_then_sends_attachment(
monkeypatch: pytest.MonkeyPatch, real_file: str
) -> None:
_patch_safe_path(monkeypatch)
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
async def _fake_cache(url: str, *a, **k) -> str:
assert url == "https://example.com/cat.jpg"
return real_file
import gateway.platforms.base as base_mod
monkeypatch.setattr(base_mod, "cache_image_from_url", _fake_cache)
result = await adapter.send_image(
"any;-;+1", "https://example.com/cat.jpg", caption="cat"
)
assert result.success is True
path, body = calls[0]
assert path == "/send-attachment"
assert body["path"] == real_file
assert body["caption"] == "cat"
@pytest.mark.asyncio
async def test_send_image_url_fetch_failure_falls_back_to_text(
monkeypatch: pytest.MonkeyPatch
) -> None:
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
async def _boom(url: str, *a, **k) -> str:
raise RuntimeError("network down")
import gateway.platforms.base as base_mod
monkeypatch.setattr(base_mod, "cache_image_from_url", _boom)
result = await adapter.send_image(
"any;-;+1", "https://example.com/cat.jpg", caption="cat"
)
# Fallback path: base send_image() routes to send() → /send (text).
assert result.success is True
assert calls[0][0] == "/send"
assert "https://example.com/cat.jpg" in calls[0][1]["text"]
@pytest.mark.asyncio
async def test_send_attachment_rejects_unsafe_path(
monkeypatch: pytest.MonkeyPatch
) -> None:
# Default validation (no passthrough patch) should reject a nonexistent /
# traversal path, returning a failed SendResult without calling the sidecar.
monkeypatch.setattr(
PhotonAdapter,
"validate_media_delivery_path",
staticmethod(lambda p: None),
)
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
result = await adapter.send_image_file("any;-;+1", "/etc/passwd")
assert result.success is False
assert "unsafe" in (result.error or "")
assert calls == [] # never reached the sidecar
@pytest.mark.asyncio
async def test_standalone_send_text_then_attachments(
monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
_patch_safe_path(monkeypatch)
img = tmp_path / "a.png"
img.write_bytes(b"\x89PNG fake")
monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok")
posted: List[Tuple[str, Dict[str, Any]]] = []
class _Resp:
status_code = 200
@staticmethod
def json() -> Dict[str, Any]:
return {"ok": True, "messageId": "m-9"}
class _FakeClient:
def __init__(self, *a, **k):
pass
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
async def post(self, url: str, json: Dict[str, Any], headers=None):
posted.append((url, json))
return _Resp()
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient)
cfg = PlatformConfig(enabled=True, token="", extra={})
result = await photon_adapter._standalone_send(
cfg,
"any;-;+1",
"hello",
media_files=[(str(img), False)],
)
assert result.get("success") is True
# First call is the text /send, second is /send-attachment.
assert posted[0][0].endswith("/send")
assert posted[0][1]["text"] == "hello"
assert posted[1][0].endswith("/send-attachment")
assert posted[1][1]["path"] == str(img)
assert posted[1][1]["kind"] == "attachment"
assert posted[1][1]["mimeType"] == "image/png"
tests/plugins/platforms/photon/test_reactions.py
+275
View File
@@ -0,0 +1,275 @@
"""Reaction (tapback) tests for PhotonAdapter.
Outbound reactions go through the sidecar's ``/react`` / ``/unreact``
endpoints; these tests stub ``_sidecar_call`` to assert endpoint + body
shape. Inbound reaction events are fed straight to ``_dispatch_inbound``.
Neither path spawns the Node sidecar or binds ports.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Dict, List, Tuple
import pytest
from gateway.config import PlatformConfig
from gateway.platforms.base import MessageEvent, MessageType, ProcessingOutcome
from plugins.platforms.photon.adapter import PhotonAdapter
_EYES = "\U0001f440"
_THUMBS_UP = "\U0001f44d"
_THUMBS_DOWN = "\U0001f44e"
def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
cfg = PlatformConfig(enabled=True, token="", extra={})
return PhotonAdapter(cfg)
def _capture_sidecar(adapter: PhotonAdapter) -> List[Tuple[str, Dict[str, Any]]]:
calls: List[Tuple[str, Dict[str, Any]]] = []
async def _fake_call(path: str, body: Dict[str, Any]) -> Dict[str, Any]:
calls.append((path, body))
return {"ok": True, "messageId": "msg-123", "reactionId": "react-1"}
adapter._sidecar_call = _fake_call # type: ignore[assignment]
return calls
def _capture_handled(
adapter: PhotonAdapter, monkeypatch: pytest.MonkeyPatch
) -> List[MessageEvent]:
captured: List[MessageEvent] = []
async def fake_handle(event: MessageEvent) -> None:
captured.append(event)
monkeypatch.setattr(adapter, "handle_message", fake_handle)
return captured
def _message_event(adapter: PhotonAdapter) -> MessageEvent:
return MessageEvent(
text="hi",
message_type=MessageType.TEXT,
source=adapter.build_source(
chat_id="+15551234567",
chat_name="+15551234567",
chat_type="dm",
user_id="+15551234567",
user_name=None,
),
message_id="target-msg-1",
timestamp=datetime.now(tz=timezone.utc),
)
def _reaction_event(
emoji: str = "❤️",
target_id: str = "bot-msg-1",
target_direction: Any = "outbound",
space_type: str = "dm",
) -> Dict[str, Any]:
return {
"messageId": "reaction-evt-1",
"platform": "iMessage",
"space": {"id": "+15551234567", "type": space_type, "phone": "+15551234567"},
"sender": {"id": "+15551234567"},
"content": {
"type": "reaction",
"emoji": emoji,
"targetMessageId": target_id,
"targetDirection": target_direction,
},
"timestamp": "2026-06-11T10:00:00.000Z",
}
# -- Outbound: /react and /unreact body shapes ------------------------------
@pytest.mark.asyncio
async def test_add_reaction_posts_react(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
ok = await adapter._add_reaction("+15551234567", "target-msg-1", _EYES)
assert ok is True
assert calls == [
(
"/react",
{
"spaceId": "+15551234567",
"messageId": "target-msg-1",
"emoji": _EYES,
},
)
]
@pytest.mark.asyncio
async def test_remove_reaction_posts_unreact(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
ok = await adapter._remove_reaction("+15551234567", "target-msg-1")
assert ok is True
assert calls == [
("/unreact", {"spaceId": "+15551234567", "messageId": "target-msg-1"})
]
@pytest.mark.asyncio
async def test_reaction_failure_is_soft(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
async def _boom(path: str, body: Dict[str, Any]) -> Dict[str, Any]:
raise RuntimeError("sidecar down")
adapter._sidecar_call = _boom # type: ignore[assignment]
assert await adapter._add_reaction("+1", "m", _EYES) is False
assert await adapter._remove_reaction("+1", "m") is False
# -- Lifecycle hooks ---------------------------------------------------------
@pytest.mark.asyncio
async def test_hooks_noop_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("PHOTON_REACTIONS", raising=False)
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
event = _message_event(adapter)
await adapter.on_processing_start(event)
await adapter.on_processing_complete(event, ProcessingOutcome.SUCCESS)
assert calls == []
@pytest.mark.asyncio
async def test_processing_start_adds_eyes(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PHOTON_REACTIONS", "true")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.on_processing_start(_message_event(adapter))
assert len(calls) == 1
path, body = calls[0]
assert path == "/react"
assert body["emoji"] == _EYES
assert body["messageId"] == "target-msg-1"
@pytest.mark.asyncio
async def test_processing_success_swaps_to_thumbs_up(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("PHOTON_REACTIONS", "true")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.on_processing_complete(
_message_event(adapter), ProcessingOutcome.SUCCESS
)
assert [path for path, _ in calls] == ["/unreact", "/react"]
assert calls[1][1]["emoji"] == _THUMBS_UP
@pytest.mark.asyncio
async def test_processing_failure_swaps_to_thumbs_down(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("PHOTON_REACTIONS", "true")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.on_processing_complete(
_message_event(adapter), ProcessingOutcome.FAILURE
)
assert [path for path, _ in calls] == ["/unreact", "/react"]
assert calls[1][1]["emoji"] == _THUMBS_DOWN
@pytest.mark.asyncio
async def test_processing_cancelled_only_removes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("PHOTON_REACTIONS", "true")
adapter = _make_adapter(monkeypatch)
calls = _capture_sidecar(adapter)
await adapter.on_processing_complete(
_message_event(adapter), ProcessingOutcome.CANCELLED
)
assert [path for path, _ in calls] == ["/unreact"]
# -- Inbound reaction routing ------------------------------------------------
@pytest.mark.asyncio
async def test_inbound_reaction_on_bot_message_routed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
adapter = _make_adapter(monkeypatch)
captured = _capture_handled(adapter, monkeypatch)
await adapter._dispatch_inbound(_reaction_event(emoji="❤️"))
assert len(captured) == 1
event = captured[0]
assert event.text == "reaction:added:❤️"
assert event.message_type == MessageType.TEXT
assert event.source.chat_id == "+15551234567"
@pytest.mark.asyncio
async def test_inbound_reaction_sent_ids_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""No targetDirection from the provider — gate on our own sent-id cache."""
adapter = _make_adapter(monkeypatch)
captured = _capture_handled(adapter, monkeypatch)
adapter._record_sent_message("bot-msg-1")
await adapter._dispatch_inbound(
_reaction_event(target_id="bot-msg-1", target_direction=None)
)
assert len(captured) == 1
@pytest.mark.asyncio
async def test_inbound_reaction_on_foreign_message_dropped(
monkeypatch: pytest.MonkeyPatch,
) -> None:
adapter = _make_adapter(monkeypatch)
captured = _capture_handled(adapter, monkeypatch)
await adapter._dispatch_inbound(
_reaction_event(target_id="someone-elses-msg", target_direction=None)
)
assert captured == []
@pytest.mark.asyncio
async def test_inbound_reaction_bypasses_require_mention(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A tapback never carries a wake word — it must skip group gating."""
monkeypatch.setenv("PHOTON_REQUIRE_MENTION", "true")
adapter = _make_adapter(monkeypatch)
captured = _capture_handled(adapter, monkeypatch)
await adapter._dispatch_inbound(_reaction_event(space_type="group"))
assert len(captured) == 1
tests/plugins/platforms/photon/test_setup_access.py
+112
View File
@@ -0,0 +1,112 @@
"""Tests for `hermes photon setup`'s access auto-configuration.
`_autoconfigure_access` allowlists the operator and points the cron home
channel at their DM, writing to the per-test ~/.hermes/.env (the hermetic
HERMES_HOME fixture isolates this). It must fill only unset keys so a re-run
never clobbers a hand-tuned allowlist.
"""
from __future__ import annotations
import argparse
import pytest
from hermes_cli.config import get_env_value, save_env_value
from plugins.platforms.photon.adapter import _env_enablement
from plugins.platforms.photon import cli
def test_autoconfigure_access_fills_unset(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("PHOTON_ALLOWED_USERS", raising=False)
monkeypatch.delenv("PHOTON_HOME_CHANNEL", raising=False)
cli._autoconfigure_access("+15551234567")
assert get_env_value("PHOTON_ALLOWED_USERS") == "+15551234567"
assert get_env_value("PHOTON_HOME_CHANNEL") == "+15551234567"
def test_autoconfigure_access_preserves_existing_allowlist(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("PHOTON_ALLOWED_USERS", raising=False)
monkeypatch.delenv("PHOTON_HOME_CHANNEL", raising=False)
# A hand-tuned allowlist already in place must survive a setup re-run.
save_env_value("PHOTON_ALLOWED_USERS", "+19998887777,+15551112222")
cli._autoconfigure_access("+15551234567")
assert get_env_value("PHOTON_ALLOWED_USERS") == "+19998887777,+15551112222"
# The still-unset home channel is filled.
assert get_env_value("PHOTON_HOME_CHANNEL") == "+15551234567"
def test_env_enablement_seeds_home_channel(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PHOTON_PROJECT_ID", "project_123")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "secret_123")
monkeypatch.setenv("PHOTON_HOME_CHANNEL", "+15551234567")
monkeypatch.setenv("PHOTON_HOME_CHANNEL_NAME", "Primary DM")
seed = _env_enablement()
assert seed is not None
assert seed["home_channel"] == {
"chat_id": "+15551234567",
"name": "Primary DM",
}
def test_env_enablement_home_channel_defaults_name(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PHOTON_PROJECT_ID", "project_123")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "secret_123")
monkeypatch.setenv("PHOTON_HOME_CHANNEL", "+15551234567")
monkeypatch.delenv("PHOTON_HOME_CHANNEL_NAME", raising=False)
seed = _env_enablement()
assert seed is not None
assert seed["home_channel"] == {
"chat_id": "+15551234567",
"name": "Home",
}
def test_setup_hint_uses_gateway_service_command(monkeypatch: pytest.MonkeyPatch, capsys) -> None:
monkeypatch.setattr(cli.photon_auth, "load_photon_token", lambda: "token")
monkeypatch.setattr(cli.photon_auth, "load_dashboard_project_id", lambda: "dashboard")
monkeypatch.setattr(
cli.photon_auth,
"ensure_spectrum_enabled",
lambda token, dashboard_id: {"spectrumProjectId": "project_123"},
)
monkeypatch.setattr(
cli.photon_auth,
"regenerate_project_secret",
lambda token, dashboard_id: "secret_123",
)
monkeypatch.setattr(cli.photon_auth, "store_project_credentials", lambda **kwargs: None)
monkeypatch.setattr(
cli.photon_auth,
"register_user_if_absent",
lambda *args, **kwargs: ({"id": "user_123", "phoneNumber": "+15551234567"}, True),
)
monkeypatch.setattr(cli.photon_auth, "user_assigned_line", lambda user: "+15557654321")
monkeypatch.setattr(cli.photon_auth, "store_user_numbers", lambda **kwargs: None)
monkeypatch.setattr(cli, "_install_sidecar", lambda: 0)
rc = cli._cmd_setup(
argparse.Namespace(
project_name=None,
phone="+15551234567",
first_name=None,
last_name=None,
email=None,
no_browser=True,
skip_sidecar_install=False,
)
)
assert rc == 0
out = capsys.readouterr().out
assert "Start the gateway: hermes gateway start" in out
assert "--platform photon" not in out
tests/plugins/platforms/photon/test_sidecar_lifecycle.py
+171
View File
@@ -0,0 +1,171 @@
"""Sidecar lifecycle tests: orphan reaping and parent-death wiring.
A hard gateway exit used to leave the detached Node sidecar squatting the
loopback port with a token the next gateway run doesn't know — every
replacement spawn then died on EADDRINUSE. These tests cover the startup
reaper (`_reap_stale_sidecar`) and the stdin-pipe lifetime binding, without
spawning Node or binding ports.
"""
from __future__ import annotations
import subprocess
from typing import Any, Dict, List, Tuple
import pytest
from gateway.config import PlatformConfig
from plugins.platforms.photon import adapter as photon_adapter
from plugins.platforms.photon.adapter import PhotonAdapter
def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:
monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
cfg = PlatformConfig(enabled=True, token="", extra={})
return PhotonAdapter(cfg)
class _ProbeClient:
"""Fake httpx.AsyncClient whose /healthz probe behavior is injectable."""
connects = True
def __init__(self, *a: Any, **k: Any) -> None:
pass
async def __aenter__(self) -> "_ProbeClient":
return self
async def __aexit__(self, *a: Any) -> bool:
return False
async def post(self, *a: Any, **k: Any) -> Any:
if not self.connects:
raise photon_adapter.httpx.ConnectError("connection refused")
class _Resp:
status_code = 401 # orphan with a different token
return _Resp()
def _capture_kills(monkeypatch: pytest.MonkeyPatch) -> List[Tuple[int, int]]:
kills: List[Tuple[int, int]] = []
def _fake_kill(pid: int, sig: int) -> None:
kills.append((pid, sig))
monkeypatch.setattr(photon_adapter.os, "kill", _fake_kill)
return kills
@pytest.mark.asyncio
async def test_reap_noop_when_port_free(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
class _Refused(_ProbeClient):
connects = False
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _Refused)
kills = _capture_kills(monkeypatch)
await adapter._reap_stale_sidecar()
assert kills == []
@pytest.mark.asyncio
async def test_reap_kills_verified_orphan(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _ProbeClient)
monkeypatch.setattr(adapter, "_find_listener_pids", lambda port: [4242])
monkeypatch.setattr(adapter, "_pid_is_sidecar", lambda pid: True)
# Dies promptly on SIGTERM — no escalation expected.
monkeypatch.setattr(adapter, "_pid_alive", lambda pid: False)
kills = _capture_kills(monkeypatch)
await adapter._reap_stale_sidecar()
assert kills == [(4242, photon_adapter.signal.SIGTERM)]
@pytest.mark.asyncio
async def test_reap_escalates_to_sigkill(monkeypatch: pytest.MonkeyPatch) -> None:
adapter = _make_adapter(monkeypatch)
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _ProbeClient)
monkeypatch.setattr(adapter, "_find_listener_pids", lambda port: [4242])
monkeypatch.setattr(adapter, "_pid_is_sidecar", lambda pid: True)
monkeypatch.setattr(adapter, "_pid_alive", lambda pid: True) # ignores TERM
# No clock fakery (logging also calls time.time, which makes a fake clock
# fragile) — this test rides out the real 3s SIGTERM grace window.
kills = _capture_kills(monkeypatch)
await adapter._reap_stale_sidecar()
assert (4242, photon_adapter.signal.SIGTERM) in kills
assert (4242, photon_adapter.signal.SIGKILL) in kills
@pytest.mark.asyncio
async def test_reap_raises_for_foreign_listener(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Never signal a process whose command line isn't our sidecar."""
adapter = _make_adapter(monkeypatch)
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _ProbeClient)
monkeypatch.setattr(adapter, "_find_listener_pids", lambda port: [777])
monkeypatch.setattr(adapter, "_pid_is_sidecar", lambda pid: False)
kills = _capture_kills(monkeypatch)
with pytest.raises(RuntimeError, match="in use by another process"):
await adapter._reap_stale_sidecar()
assert kills == []
@pytest.mark.asyncio
async def test_start_sidecar_spawns_with_stdin_pipe(
monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
"""The spawn must hold a stdin pipe and enable the sidecar's EOF watch."""
adapter = _make_adapter(monkeypatch)
async def _no_reap() -> None:
pass
monkeypatch.setattr(adapter, "_reap_stale_sidecar", _no_reap)
(tmp_path / "node_modules").mkdir()
monkeypatch.setattr(photon_adapter, "_SIDECAR_DIR", tmp_path)
spawned: Dict[str, Any] = {}
class _FakeProc:
pid = 999
stdout = None
stdin = None
@staticmethod
def poll() -> None:
return None
def _fake_popen(cmd: List[str], **kwargs: Any) -> _FakeProc:
spawned["cmd"] = cmd
spawned["kwargs"] = kwargs
return _FakeProc()
monkeypatch.setattr(photon_adapter.subprocess, "Popen", _fake_popen)
class _HealthyClient(_ProbeClient):
async def post(self, *a: Any, **k: Any) -> Any:
class _Resp:
status_code = 200
return _Resp()
monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _HealthyClient)
await adapter._start_sidecar()
kwargs = spawned["kwargs"]
assert kwargs["stdin"] is subprocess.PIPE
assert kwargs["env"]["PHOTON_SIDECAR_WATCH_STDIN"] == "1"
tests/plugins/test_achievements_plugin.py
+378
View File
@@ -0,0 +1,378 @@
"""Tests for the bundled hermes-achievements dashboard plugin.
These target the two behaviors that matter for official integration:
* The 200-session scan cap is removed — the plugin now walks the entire
session history by default. Lifetime badges (tens of thousands of
tool calls) were unreachable before this fix on long-running installs.
* First-ever scans run in a background thread so the dashboard request
path never blocks, even on 8000+ session databases where a cold scan
takes minutes.
The upstream repo ships its own unittest suite under
``plugins/hermes-achievements/tests/`` covering the achievement engine
internals (tier math, secret-state handling, catalog invariants). These
tests live at the hermes-agent level and focus on the integration
contract: the plugin scans ALL of your sessions, not the first 200.
"""
from __future__ import annotations
import importlib.util
import sys
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
import pytest
PLUGIN_MODULE_PATH = (
Path(__file__).resolve().parents[2]
/ "plugins"
/ "hermes-achievements"
/ "dashboard"
/ "plugin_api.py"
)
@pytest.fixture
def plugin_api(tmp_path, monkeypatch):
"""Load plugin_api with isolated ~/.hermes so state/snapshot files don't collide.
We load the module fresh per test because the plugin keeps module-level
caches (``_SNAPSHOT_CACHE``, ``_SCAN_STATUS``, background thread handle).
Reloading gives each test a clean world.
"""
monkeypatch.setattr(Path, "home", lambda: tmp_path)
spec = importlib.util.spec_from_file_location(
f"plugin_api_test_{id(tmp_path)}", PLUGIN_MODULE_PATH
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# Stash monkeypatch so ``_install_fake_session_db`` can use it to
# swap ``sys.modules['hermes_state']`` with auto-restoration. Without
# this, a raw ``sys.modules[...] = fake`` assignment would leak the
# fake into later tests in the same xdist worker — breaking every
# test that does ``from hermes_state import SessionDB``.
module._test_monkeypatch = monkeypatch
yield module
class _FakeSessionDB:
"""Stand-in for hermes_state.SessionDB that records scan calls."""
def __init__(self, session_count: int, scan_delay: float = 0):
self.session_count = session_count
self.scan_delay = scan_delay
self.last_limit: Optional[int] = None
self.last_include_children: Optional[bool] = None
self.list_calls = 0
self.messages_calls = 0
def list_sessions_rich(
self,
source: Optional[str] = None,
exclude_sources: Optional[List[str]] = None,
limit: int = 20,
offset: int = 0,
include_children: bool = False,
project_compression_tips: bool = True,
) -> List[Dict[str, Any]]:
if self.scan_delay:
time.sleep(self.scan_delay)
self.last_limit = limit
self.last_include_children = include_children
self.list_calls += 1
# SQLite semantics: LIMIT -1 = unlimited. Honor that here.
effective = self.session_count if limit == -1 else min(self.session_count, limit)
now = int(time.time())
return [
{
"id": f"sess-{i}",
"title": f"Session {i}",
"preview": f"preview {i}",
"started_at": now - (self.session_count - i) * 60,
"last_active": now - (self.session_count - i) * 60 + 30,
"source": "cli",
"model": "test-model",
}
for i in range(effective)
]
def get_messages(self, session_id: str) -> List[Dict[str, Any]]:
self.messages_calls += 1
return [
{"role": "user", "content": f"ask {session_id}"},
{
"role": "assistant",
"tool_calls": [{"function": {"name": "terminal"}}],
},
{"role": "tool", "tool_name": "terminal", "content": "ok"},
]
def close(self) -> None:
pass
def _install_fake_session_db(plugin_api, fake_db):
"""Inject a fake SessionDB so ``scan_sessions`` finds it via its local import.
Uses the monkeypatch stashed on ``plugin_api`` by the fixture, so the
``sys.modules['hermes_state']`` swap is auto-restored at test teardown
and cannot leak into unrelated tests in the same xdist worker.
"""
fake_module = type(sys)("hermes_state")
fake_module.SessionDB = lambda: fake_db
plugin_api._test_monkeypatch.setitem(sys.modules, "hermes_state", fake_module)
def test_scan_sessions_default_scans_all_history_not_first_200(plugin_api):
"""Bug regression: ``scan_sessions()`` used to cap at limit=200.
A user with 8000+ sessions would only see ~2% of their history in
achievement totals, making lifetime badges unreachable. The default
now passes ``LIMIT -1`` (SQLite "unlimited") to ``list_sessions_rich``.
"""
fake_db = _FakeSessionDB(session_count=500) # > old 200 cap
_install_fake_session_db(plugin_api, fake_db)
result = plugin_api.scan_sessions()
assert fake_db.last_limit == -1, (
"scan_sessions() must pass LIMIT=-1 (unlimited) to list_sessions_rich "
f"by default, got {fake_db.last_limit}"
)
assert fake_db.last_include_children is True, (
"scan_sessions() must include subagent/compression child sessions so "
"tool calls made in delegated agents still count toward achievements"
)
assert len(result["sessions"]) == 500
assert result["scan_meta"]["sessions_total"] == 500
def test_scan_sessions_explicit_positive_limit_is_honored(plugin_api):
"""Callers can still pass a small limit for smoke tests."""
fake_db = _FakeSessionDB(session_count=500)
_install_fake_session_db(plugin_api, fake_db)
result = plugin_api.scan_sessions(limit=10)
assert fake_db.last_limit == 10
assert len(result["sessions"]) == 10
def test_scan_sessions_zero_or_negative_limit_means_unlimited(plugin_api):
"""``limit=0`` and ``limit=-1`` both map to the unlimited path."""
fake_db = _FakeSessionDB(session_count=300)
_install_fake_session_db(plugin_api, fake_db)
plugin_api.scan_sessions(limit=0)
assert fake_db.last_limit == -1
plugin_api.scan_sessions(limit=-1)
assert fake_db.last_limit == -1
def test_evaluate_all_first_run_returns_pending_and_starts_background_scan(plugin_api):
"""First-ever evaluate_all with no cache returns a pending placeholder
immediately and kicks off a background scan thread. Cold scans on
large DBs take minutes — blocking the dashboard request path is not
acceptable.
"""
fake_db = _FakeSessionDB(session_count=50)
_install_fake_session_db(plugin_api, fake_db)
# Wrap _run_scan_and_update_cache so we can release it on demand,
# simulating a slow cold scan without actually waiting.
scan_started = threading.Event()
allow_scan_finish = threading.Event()
original_run = plugin_api._run_scan_and_update_cache
def gated_run(*args, **kwargs):
scan_started.set()
allow_scan_finish.wait(timeout=5)
original_run(*args, **kwargs)
plugin_api._run_scan_and_update_cache = gated_run
t0 = time.time()
result = plugin_api.evaluate_all()
elapsed = time.time() - t0
# Immediate return — should not block waiting for the scan.
assert elapsed < 1.0, f"evaluate_all blocked for {elapsed:.2f}s on first run"
assert result["scan_meta"]["mode"] == "pending"
assert result["unlocked_count"] == 0
# Catalog still rendered so UI has something to draw.
assert result["total_count"] >= 60
# Background scan is running.
assert scan_started.wait(timeout=2), "background scan did not start"
# Let the scan complete, then a second call returns real data.
allow_scan_finish.set()
# Wait for thread to finish.
thread = plugin_api._BACKGROUND_SCAN_THREAD
assert thread is not None
thread.join(timeout=5)
assert not thread.is_alive()
second = plugin_api.evaluate_all()
assert second["scan_meta"]["mode"] != "pending"
assert second["scan_meta"].get("sessions_total") == 50
def test_evaluate_all_stale_cache_serves_stale_and_refreshes_in_background(plugin_api):
"""When the snapshot is on-disk but older than TTL, evaluate_all returns
the stale data immediately and kicks a background refresh. Users don't
stare at a loading spinner every time TTL expires.
"""
fake_db = _FakeSessionDB(session_count=10, scan_delay=2.0)
_install_fake_session_db(plugin_api, fake_db)
stale_generated_at = int(time.time()) - plugin_api.SNAPSHOT_TTL_SECONDS - 60
stale_payload = {
"achievements": [],
"sessions": [],
"aggregate": {},
"scan_meta": {"mode": "full", "sessions_total": 1, "sessions_rescanned": 1, "sessions_reused": 0},
"error": None,
"unlocked_count": 0,
"discovered_count": 0,
"secret_count": 0,
"total_count": 0,
"generated_at": stale_generated_at,
}
plugin_api.save_snapshot(stale_payload)
t0 = time.time()
result = plugin_api.evaluate_all()
elapsed = time.time() - t0
assert elapsed < 1.0, f"evaluate_all blocked for {elapsed:.2f}s serving stale data"
assert result["generated_at"] == stale_generated_at
# Background scan should be running or have completed.
thread = plugin_api._BACKGROUND_SCAN_THREAD
assert thread is not None
thread.join(timeout=5)
fresh = plugin_api.evaluate_all()
assert fresh["generated_at"] >= stale_generated_at
def test_evaluate_all_force_runs_synchronously(plugin_api):
"""Manual /rescan (force=True) blocks the caller — users clicking
the rescan button expect up-to-date data when the call returns.
"""
fake_db = _FakeSessionDB(session_count=25)
_install_fake_session_db(plugin_api, fake_db)
result = plugin_api.evaluate_all(force=True)
# Synchronous — snapshot is fresh on return.
assert result["scan_meta"].get("sessions_total") == 25
assert result["scan_meta"]["mode"] in {"full", "incremental"}
def test_start_background_scan_is_idempotent_while_running(plugin_api):
"""Multiple concurrent dashboard requests must not spawn duplicate scans."""
fake_db = _FakeSessionDB(session_count=5)
_install_fake_session_db(plugin_api, fake_db)
release = threading.Event()
original_run = plugin_api._run_scan_and_update_cache
def gated_run(*args, **kwargs):
release.wait(timeout=5)
original_run(*args, **kwargs)
plugin_api._run_scan_and_update_cache = gated_run
plugin_api._start_background_scan()
first_thread = plugin_api._BACKGROUND_SCAN_THREAD
assert first_thread is not None and first_thread.is_alive()
plugin_api._start_background_scan()
plugin_api._start_background_scan()
assert plugin_api._BACKGROUND_SCAN_THREAD is first_thread
release.set()
first_thread.join(timeout=5)
def test_background_scan_publishes_partial_snapshots(plugin_api):
"""The background scanner publishes intermediate snapshots to the cache
every ~N sessions. Each dashboard refresh during a long cold scan sees
more badges unlocked instead of staring at zeros for minutes and then
having everything pop at the end.
"""
fake_db = _FakeSessionDB(session_count=750)
_install_fake_session_db(plugin_api, fake_db)
# Record every partial snapshot the scanner publishes.
partial_snapshots: List[Dict[str, Any]] = []
original_compute_from_scan = plugin_api._compute_from_scan
def recording_compute(scan, *, is_partial=False):
result = original_compute_from_scan(scan, is_partial=is_partial)
if is_partial:
partial_snapshots.append(result)
return result
plugin_api._compute_from_scan = recording_compute
# scan 750 sessions with progress_every=250 → expect 2 intermediate
# publications (at 250 and 500; the final 750 call goes through the
# finished, non-partial path).
plugin_api._run_scan_and_update_cache(publish_partial_snapshots=True)
assert len(partial_snapshots) >= 2, (
f"expected at least 2 partial publications on a 750-session scan with "
f"progress_every=250, got {len(partial_snapshots)}"
)
# Partial snapshots should report growing session counts.
counts = [p["scan_meta"].get("sessions_scanned_so_far") for p in partial_snapshots]
assert counts == sorted(counts), f"partial session counts not monotonic: {counts}"
assert counts[0] < 750 and counts[-1] < 750, (
f"partial counts should be less than the final total; got {counts}"
)
# Every partial reports the expected end-state total so the UI can
# show an accurate progress bar.
for p in partial_snapshots:
assert p["scan_meta"].get("sessions_expected_total") == 750
# Final snapshot in cache is the real (non-partial) one.
final = plugin_api._SNAPSHOT_CACHE
assert final is not None
assert final["scan_meta"].get("mode") != "in_progress"
assert final["scan_meta"].get("sessions_total") == 750
def test_partial_snapshots_do_not_persist_unlock_timestamps(plugin_api):
"""Intermediate snapshots must not write to state.json — an unlock
that appears at 30% scan progress could disappear when a later session
rebalances the aggregate. Only the final snapshot records ``unlocked_at``.
"""
fake_db = _FakeSessionDB(session_count=10)
_install_fake_session_db(plugin_api, fake_db)
# Seed empty state, then invoke partial compute directly.
plugin_api.save_state({"unlocks": {}})
partial_scan = {
"sessions": [{"session_id": "x", "tool_call_count": 99999, "tool_names": set()}],
"aggregate": {"max_tool_calls_in_session": 99999, "total_tool_calls": 99999},
"scan_meta": {"mode": "in_progress"},
}
result = plugin_api._compute_from_scan(partial_scan, is_partial=True)
# Some achievements should evaluate as unlocked in this aggregate...
assert any(a["unlocked"] for a in result["achievements"])
# ...but state.json on disk stays empty (no timestamps were recorded).
persisted = plugin_api.load_state()
assert persisted.get("unlocks", {}) == {}, (
"partial scans must not record unlock timestamps — a later session "
"could change whether the badge deserves to be unlocked yet"
)
tests/plugins/test_discord_runtime_failure.py
+59
View File
@@ -0,0 +1,59 @@
import asyncio
from unittest.mock import AsyncMock
import pytest
from gateway.config import PlatformConfig
from plugins.platforms.discord.adapter import DiscordAdapter
@pytest.mark.asyncio
async def test_discord_bot_task_runtime_exit_notifies_gateway_for_reconnect(monkeypatch):
"""A post-ready discord.py websocket task crash must not leave the gateway split-brained.
Regression: producers stayed systemd-active while Discord stopped responding after
a runtime ClientOSError/ConnectionResetError. The adapter must mark Discord as a
retryable fatal platform error and notify the gateway supervisor so the existing
reconnect watcher can replace the dead adapter.
"""
adapter = DiscordAdapter(PlatformConfig(enabled=True, token="token"))
adapter._running = True
adapter._ready_event.set()
adapter._notify_fatal_error = AsyncMock()
async def crash():
raise ConnectionResetError("Cannot write to closing transport")
task = asyncio.create_task(crash())
await asyncio.sleep(0)
adapter._handle_bot_task_done(task)
await asyncio.sleep(0)
assert adapter.has_fatal_error is True
assert adapter.fatal_error_retryable is True
assert adapter.fatal_error_code == "discord_gateway_task_exited"
assert adapter.fatal_error_message is not None
assert "Cannot write to closing transport" in adapter.fatal_error_message
adapter._notify_fatal_error.assert_awaited_once()
@pytest.mark.asyncio
async def test_discord_bot_task_done_ignored_during_intentional_disconnect():
adapter = DiscordAdapter(PlatformConfig(enabled=True, token="token"))
adapter._running = True
adapter._ready_event.set()
adapter._disconnecting = True
adapter._notify_fatal_error = AsyncMock()
async def stop_cleanly():
return None
task = asyncio.create_task(stop_cleanly())
await asyncio.sleep(0)
adapter._handle_bot_task_done(task)
await asyncio.sleep(0)
assert adapter.has_fatal_error is False
adapter._notify_fatal_error.assert_not_awaited()
tests/plugins/test_disk_cleanup_plugin.py
+584
View File
@@ -0,0 +1,584 @@
"""Tests for the disk-cleanup plugin.
Covers the bundled plugin at ``plugins/disk-cleanup/``:
* ``disk_cleanup`` library: track / forget / dry_run / quick / status,
``is_safe_path`` and ``guess_category`` filtering.
* Plugin ``__init__``: ``post_tool_call`` hook auto-tracks files created
by ``write_file`` / ``terminal``; ``on_session_end`` hook runs quick
cleanup when anything was tracked during the turn.
* Slash command handler: status / dry-run / quick / track / forget /
unknown subcommand behaviours.
* Bundled-plugin discovery via ``PluginManager.discover_and_load``.
"""
import importlib
import json
import sys
from pathlib import Path
import pytest
@pytest.fixture(autouse=True)
def _isolate_env(tmp_path, monkeypatch):
"""Isolate HERMES_HOME for each test.
The global hermetic fixture already redirects HERMES_HOME to a tempdir,
but we want the plugin to work with a predictable subpath. We reset
HERMES_HOME here for clarity.
"""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
def _load_lib():
"""Import the plugin's library module directly from the repo path."""
repo_root = Path(__file__).resolve().parents[2]
lib_path = repo_root / "plugins" / "disk-cleanup" / "disk_cleanup.py"
spec = importlib.util.spec_from_file_location(
"disk_cleanup_under_test", lib_path
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _load_plugin_init():
"""Import the plugin's __init__.py (which depends on the library)."""
repo_root = Path(__file__).resolve().parents[2]
plugin_dir = repo_root / "plugins" / "disk-cleanup"
# Use the PluginManager's module naming convention so relative imports work.
spec = importlib.util.spec_from_file_location(
"hermes_plugins.disk_cleanup",
plugin_dir / "__init__.py",
submodule_search_locations=[str(plugin_dir)],
)
# Ensure parent namespace package exists for the relative `. import disk_cleanup`
import types
if "hermes_plugins" not in sys.modules:
ns = types.ModuleType("hermes_plugins")
ns.__path__ = []
sys.modules["hermes_plugins"] = ns
mod = importlib.util.module_from_spec(spec)
mod.__package__ = "hermes_plugins.disk_cleanup"
mod.__path__ = [str(plugin_dir)]
sys.modules["hermes_plugins.disk_cleanup"] = mod
spec.loader.exec_module(mod)
return mod
# ---------------------------------------------------------------------------
# Library tests
# ---------------------------------------------------------------------------
class TestIsSafePath:
def test_accepts_path_under_hermes_home(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "subdir" / "file.txt"
p.parent.mkdir()
p.write_text("x")
assert dg.is_safe_path(p) is True
def test_rejects_outside_hermes_home(self, _isolate_env):
dg = _load_lib()
assert dg.is_safe_path(Path("/etc/passwd")) is False
def test_accepts_tmp_hermes_prefix(self, _isolate_env, tmp_path):
dg = _load_lib()
assert dg.is_safe_path(Path("/tmp/hermes-abc/x.log")) is True
def test_rejects_plain_tmp(self, _isolate_env):
dg = _load_lib()
assert dg.is_safe_path(Path("/tmp/other.log")) is False
def test_rejects_windows_mount(self, _isolate_env):
dg = _load_lib()
assert dg.is_safe_path(Path("/mnt/c/Users/x/test.txt")) is False
class TestGuessCategory:
def test_test_prefix(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "test_foo.py"
p.write_text("x")
assert dg.guess_category(p) == "test"
def test_tmp_prefix(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "tmp_foo.log"
p.write_text("x")
assert dg.guess_category(p) == "test"
def test_dot_test_suffix(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "mything.test.js"
p.write_text("x")
assert dg.guess_category(p) == "test"
def test_skips_protected_top_level(self, _isolate_env):
dg = _load_lib()
logs_dir = _isolate_env / "logs"
logs_dir.mkdir()
p = logs_dir / "test_log.txt"
p.write_text("x")
# Even though it matches test_* pattern, logs/ is excluded.
assert dg.guess_category(p) is None
def test_cron_subtree_categorised(self, _isolate_env):
dg = _load_lib()
# Only files under ``cron/output/`` are disposable run artifacts.
output_dir = _isolate_env / "cron" / "output" / "job_123"
output_dir.mkdir(parents=True)
p = output_dir / "run.md"
p.write_text("x")
assert dg.guess_category(p) == "cron-output"
def test_cron_jobs_json_not_tracked(self, _isolate_env):
"""Regression for #32164: the cron registry must never be tracked."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
p = cron_dir / "jobs.json"
p.write_text("[]")
assert dg.guess_category(p) is None
def test_cron_tick_lock_not_tracked(self, _isolate_env):
"""Regression for #32164: cron tick-lock is control-plane state."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
p = cron_dir / ".tick.lock"
p.write_text("")
assert dg.guess_category(p) is None
def test_cronjobs_top_level_not_tracked(self, _isolate_env):
"""The legacy ``cronjobs`` alias is also control-plane at the top."""
dg = _load_lib()
cron_dir = _isolate_env / "cronjobs"
cron_dir.mkdir()
p = cron_dir / "jobs.json"
p.write_text("[]")
assert dg.guess_category(p) is None
def test_ordinary_file_returns_none(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "notes.md"
p.write_text("x")
assert dg.guess_category(p) is None
class TestStaleCronEntryMigration:
"""Regression tests for #37721 — stale cron-output entries in tracked.json."""
def test_quick_skips_stale_cron_output_for_jobs_json(self, _isolate_env):
"""A stale tracked.json entry with category="cron-output" for
cron/jobs.json must NOT be deleted by quick().
This is the exact scenario from #37721: an old tracked.json has
{"path": ".../cron/jobs.json", "category": "cron-output"} which
would pass the delete filter but must be skipped because
guess_category() now returns None for non-output cron paths.
"""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
jobs_json = cron_dir / "jobs.json"
jobs_json.write_text('{"jobs": []}')
# Simulate a stale tracked.json entry from before #34840 by
# directly writing the tracked file (track() would reject it).
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
tracked_file.parent.mkdir(parents=True, exist_ok=True)
tracked_file.write_text(json.dumps([{
"path": str(jobs_json),
"category": "cron-output",
"timestamp": "2025-01-01T00:00:00+00:00", # very old
"size": 123,
}]))
summary = dg.quick()
assert summary["deleted"] == 0, "cron/jobs.json must not be deleted"
assert jobs_json.exists(), "jobs.json must still exist"
# The stale entry should have been dropped from tracking.
remaining = json.loads(tracked_file.read_text())
assert len(remaining) == 0
def test_quick_skips_stale_cron_output_for_cron_dir(self, _isolate_env):
"""Stale entry for the cron/ directory itself must not be deleted."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
output_dir = cron_dir / "output"
output_dir.mkdir()
(output_dir / "run.md").write_text("x")
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
tracked_file.parent.mkdir(parents=True, exist_ok=True)
tracked_file.write_text(json.dumps([{
"path": str(cron_dir),
"category": "cron-output",
"timestamp": "2025-01-01T00:00:00+00:00",
"size": 0,
}]))
summary = dg.quick()
assert summary["deleted"] == 0, "cron/ dir must not be deleted"
assert cron_dir.exists()
def test_quick_skips_protected_cron_paths_defense_in_depth(self, _isolate_env):
"""Defense-in-depth: even if guess_category returned cron-output
(hypothetically), protected cron paths are never deleted."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
tick_lock = cron_dir / ".tick.lock"
tick_lock.write_text("")
# Manually inject a stale entry with "test" category (would normally
# be auto-deleted) — the protected path guard must still block it.
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
tracked_file.parent.mkdir(parents=True, exist_ok=True)
tracked_file.write_text(json.dumps([{
"path": str(tick_lock),
"category": "test",
"timestamp": "2025-01-01T00:00:00+00:00",
"size": 0,
}]))
summary = dg.quick()
assert summary["deleted"] == 0, ".tick.lock must not be deleted"
assert tick_lock.exists()
def test_dry_run_omits_stale_cron_output(self, _isolate_env):
"""dry_run() should also skip stale cron-output entries."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
jobs_json = cron_dir / "jobs.json"
jobs_json.write_text("[]")
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
tracked_file.parent.mkdir(parents=True, exist_ok=True)
tracked_file.write_text(json.dumps([{
"path": str(jobs_json),
"category": "cron-output",
"timestamp": "2025-01-01T00:00:00+00:00",
"size": 123,
}]))
auto, prompt = dg.dry_run()
assert len(auto) == 0, "stale cron-output for jobs.json must not appear"
assert len(prompt) == 0
def test_legitimate_cron_output_still_deleted(self, _isolate_env):
"""A valid cron-output entry under cron/output/ must still be deleted."""
dg = _load_lib()
output_dir = _isolate_env / "cron" / "output" / "job_1"
output_dir.mkdir(parents=True)
run_md = output_dir / "run.md"
run_md.write_text("x")
# Old enough to be deleted (>14 days)
from datetime import datetime, timezone, timedelta
old_ts = (datetime.now(timezone.utc) - timedelta(days=20)).isoformat()
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
tracked_file.parent.mkdir(parents=True, exist_ok=True)
tracked_file.write_text(json.dumps([{
"path": str(run_md),
"category": "cron-output",
"timestamp": old_ts,
"size": 10,
}]))
summary = dg.quick()
assert summary["deleted"] == 1, "valid old cron-output should be deleted"
assert not run_md.exists()
class TestTrackForgetQuick:
def test_track_then_quick_deletes_test(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "test_a.py"
p.write_text("x")
assert dg.track(str(p), "test", silent=True) is True
summary = dg.quick()
assert summary["deleted"] == 1
assert not p.exists()
def test_track_dedup(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "test_a.py"
p.write_text("x")
assert dg.track(str(p), "test", silent=True) is True
# Second call returns False (already tracked)
assert dg.track(str(p), "test", silent=True) is False
def test_track_rejects_outside_home(self, _isolate_env):
dg = _load_lib()
# /etc/hostname exists on most Linux boxes; fall back if not.
outside = "/etc/hostname" if Path("/etc/hostname").exists() else "/etc/passwd"
assert dg.track(outside, "test", silent=True) is False
def test_track_skips_missing(self, _isolate_env):
dg = _load_lib()
assert dg.track(str(_isolate_env / "nope.txt"), "test", silent=True) is False
def test_forget_removes_entry(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "keep.tmp"
p.write_text("x")
dg.track(str(p), "temp", silent=True)
assert dg.forget(str(p)) == 1
assert p.exists() # forget does NOT delete the file
def test_quick_preserves_unexpired_temp(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "fresh.tmp"
p.write_text("x")
dg.track(str(p), "temp", silent=True)
summary = dg.quick()
assert summary["deleted"] == 0
assert p.exists()
def test_quick_preserves_protected_top_level_dirs(self, _isolate_env):
dg = _load_lib()
for d in ("logs", "memories", "sessions", "cron", "cache"):
(_isolate_env / d).mkdir()
dg.quick()
for d in ("logs", "memories", "sessions", "cron", "cache"):
assert (_isolate_env / d).exists(), f"{d}/ should be preserved"
class TestStatus:
def test_empty_status(self, _isolate_env):
dg = _load_lib()
s = dg.status()
assert s["total_tracked"] == 0
assert s["top10"] == []
def test_status_with_entries(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "big.tmp"
p.write_text("y" * 100)
dg.track(str(p), "temp", silent=True)
s = dg.status()
assert s["total_tracked"] == 1
assert len(s["top10"]) == 1
rendered = dg.format_status(s)
assert "temp" in rendered
assert "big.tmp" in rendered
class TestDryRun:
def test_classifies_by_category(self, _isolate_env):
dg = _load_lib()
test_f = _isolate_env / "test_x.py"
test_f.write_text("x")
big = _isolate_env / "big.bin"
big.write_bytes(b"z" * 10)
dg.track(str(test_f), "test", silent=True)
dg.track(str(big), "other", silent=True)
auto, prompt = dg.dry_run()
# test → auto, other → neither (doesn't hit any rule)
assert any(i["path"] == str(test_f) for i in auto)
# ---------------------------------------------------------------------------
# Plugin hooks tests
# ---------------------------------------------------------------------------
class TestPostToolCallHook:
def test_write_file_test_pattern_tracked(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "test_created.py"
p.write_text("x")
pi._on_post_tool_call(
tool_name="write_file",
args={"path": str(p), "content": "x"},
result="OK",
task_id="t1", session_id="s1",
)
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
data = json.loads(tracked_file.read_text())
assert len(data) == 1
assert data[0]["category"] == "test"
def test_write_file_non_test_not_tracked(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "notes.md"
p.write_text("x")
pi._on_post_tool_call(
tool_name="write_file",
args={"path": str(p), "content": "x"},
result="OK",
task_id="t2", session_id="s2",
)
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
assert not tracked_file.exists() or tracked_file.read_text().strip() == "[]"
def test_terminal_command_picks_up_paths(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "tmp_created.log"
p.write_text("x")
pi._on_post_tool_call(
tool_name="terminal",
args={"command": f"touch {p}"},
result=f"created {p}\n",
task_id="t3", session_id="s3",
)
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
data = json.loads(tracked_file.read_text())
assert any(Path(i["path"]) == p.resolve() for i in data)
def test_ignores_unrelated_tool(self, _isolate_env):
pi = _load_plugin_init()
pi._on_post_tool_call(
tool_name="read_file",
args={"path": str(_isolate_env / "test_x.py")},
result="contents",
task_id="t4", session_id="s4",
)
# read_file should never trigger tracking.
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
assert not tracked_file.exists() or tracked_file.read_text().strip() == "[]"
class TestOnSessionEndHook:
def test_runs_quick_when_test_files_tracked(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "test_cleanup.py"
p.write_text("x")
pi._on_post_tool_call(
tool_name="write_file",
args={"path": str(p), "content": "x"},
result="OK",
task_id="", session_id="s1",
)
assert p.exists()
pi._on_session_end(session_id="s1", completed=True, interrupted=False)
assert not p.exists(), "test file should be auto-deleted"
def test_noop_when_no_test_tracked(self, _isolate_env):
pi = _load_plugin_init()
# Nothing tracked → on_session_end should not raise.
pi._on_session_end(session_id="empty", completed=True, interrupted=False)
# ---------------------------------------------------------------------------
# Slash command
# ---------------------------------------------------------------------------
class TestSlashCommand:
def test_help(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("help")
assert "disk-cleanup" in out
assert "status" in out
def test_status_empty(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("status")
assert "nothing tracked" in out
def test_track_rejects_missing(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash(
f"track {_isolate_env / 'nope.txt'} temp"
)
assert "Not tracked" in out
def test_track_rejects_bad_category(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "a.tmp"
p.write_text("x")
out = pi._handle_slash(f"track {p} banana")
assert "Unknown category" in out
def test_track_and_forget(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "a.tmp"
p.write_text("x")
out = pi._handle_slash(f"track {p} temp")
assert "Tracked" in out
out = pi._handle_slash(f"forget {p}")
assert "Removed 1" in out
def test_unknown_subcommand(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("foobar")
assert "Unknown subcommand" in out
def test_quick_on_empty(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("quick")
assert "Cleaned 0 files" in out
# ---------------------------------------------------------------------------
# Bundled-plugin discovery
# ---------------------------------------------------------------------------
class TestBundledDiscovery:
def _write_enabled_config(self, hermes_home, names):
"""Write plugins.enabled allow-list to config.yaml."""
import yaml
cfg_path = hermes_home / "config.yaml"
cfg_path.write_text(yaml.safe_dump({"plugins": {"enabled": list(names)}}))
def test_disk_cleanup_discovered_but_not_loaded_by_default(self, _isolate_env):
"""Bundled plugins are discovered but NOT loaded without opt-in."""
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
# Discovered — appears in the registry
assert "disk-cleanup" in mgr._plugins
loaded = mgr._plugins["disk-cleanup"]
assert loaded.manifest.source == "bundled"
# But NOT enabled — no hooks or commands registered
assert not loaded.enabled
assert loaded.error and "not enabled" in loaded.error
def test_disk_cleanup_loads_when_enabled(self, _isolate_env):
"""Adding to plugins.enabled activates the bundled plugin."""
self._write_enabled_config(_isolate_env, ["disk-cleanup"])
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
loaded = mgr._plugins["disk-cleanup"]
assert loaded.enabled
assert "post_tool_call" in loaded.hooks_registered
assert "on_session_end" in loaded.hooks_registered
assert "disk-cleanup" in loaded.commands_registered
def test_disabled_beats_enabled(self, _isolate_env):
"""plugins.disabled wins even if the plugin is also in plugins.enabled."""
import yaml
cfg_path = _isolate_env / "config.yaml"
cfg_path.write_text(yaml.safe_dump({
"plugins": {
"enabled": ["disk-cleanup"],
"disabled": ["disk-cleanup"],
}
}))
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
loaded = mgr._plugins["disk-cleanup"]
assert not loaded.enabled
assert loaded.error == "disabled via config"
def test_memory_and_context_engine_subdirs_skipped(self, _isolate_env):
"""Bundled scan must NOT pick up plugins/memory or plugins/context_engine
as top-level plugins — they have their own discovery paths."""
self._write_enabled_config(
_isolate_env, ["memory", "context_engine", "disk-cleanup"]
)
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
assert "memory" not in mgr._plugins
assert "context_engine" not in mgr._plugins
tests/plugins/test_google_meet_audio.py
+265
View File
@@ -0,0 +1,265 @@
"""Tests for plugins.google_meet.audio_bridge (v2).
Covers the platform gating and pactl / system_profiler plumbing
without actually invoking those tools on the host.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# Linux setup / teardown
# ---------------------------------------------------------------------------
def _linux_pactl_result(stdout: str) -> MagicMock:
"""Build a fake CompletedProcess-ish object for subprocess.run."""
m = MagicMock()
m.stdout = stdout
m.stderr = ""
m.returncode = 0
return m
def test_setup_linux_loads_null_sink_and_virtual_source():
from plugins.google_meet.audio_bridge import AudioBridge
calls: list[list[str]] = []
def _fake_run(argv, **kwargs):
calls.append(list(argv))
# First call = null-sink → module id 42
# Second call = virtual-source → module id 43
if "module-null-sink" in argv:
return _linux_pactl_result("42\n")
if "module-virtual-source" in argv:
return _linux_pactl_result("43\n")
raise AssertionError(f"unexpected pactl invocation: {argv}")
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_fake_run):
br = AudioBridge()
info = br.setup()
# Two pactl load-module calls, in order.
assert len(calls) == 2
assert calls[0][0] == "pactl" and calls[0][1] == "load-module"
assert "module-null-sink" in calls[0]
assert any(a.startswith("sink_name=hermes_meet_sink") for a in calls[0])
assert calls[1][0] == "pactl" and calls[1][1] == "load-module"
assert "module-virtual-source" in calls[1]
assert any(a.startswith("source_name=hermes_meet_src") for a in calls[1])
assert any("master=hermes_meet_sink.monitor" in a for a in calls[1])
# Dict shape.
assert info["platform"] == "linux"
assert info["device_name"] == "hermes_meet_src"
assert info["write_target"] == "hermes_meet_sink"
assert info["sample_rate"] == 48000
assert info["channels"] == 2
assert info["module_ids"] == [42, 43]
# Properties.
assert br.device_name == "hermes_meet_src"
assert br.write_target == "hermes_meet_sink"
def test_teardown_linux_unloads_modules_in_reverse_order():
from plugins.google_meet.audio_bridge import AudioBridge
def _setup_run(argv, **kwargs):
if "module-null-sink" in argv:
return _linux_pactl_result("42\n")
return _linux_pactl_result("43\n")
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_setup_run):
br = AudioBridge()
br.setup()
unload_calls: list[list[str]] = []
def _teardown_run(argv, **kwargs):
unload_calls.append(list(argv))
return _linux_pactl_result("")
with patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_teardown_run):
br.teardown()
# Two unload calls, in reverse order: 43 (virtual-source) then 42 (sink).
assert [c[1] for c in unload_calls] == ["unload-module", "unload-module"]
assert unload_calls[0][2] == "43"
assert unload_calls[1][2] == "42"
# Second teardown is a no-op.
with patch("plugins.google_meet.audio_bridge.subprocess.run") as run_mock:
br.teardown()
run_mock.assert_not_called()
def test_setup_linux_parses_module_id_from_multi_line_output():
"""Some pactl builds include trailing whitespace / notices."""
from plugins.google_meet.audio_bridge import AudioBridge
def _fake_run(argv, **kwargs):
if "module-null-sink" in argv:
return _linux_pactl_result("42 \n")
return _linux_pactl_result("43\n")
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_fake_run):
br = AudioBridge()
info = br.setup()
assert info["module_ids"] == [42, 43]
def test_setup_linux_pactl_missing_raises_clean_error():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=FileNotFoundError("pactl")):
br = AudioBridge()
with pytest.raises(RuntimeError, match="pactl"):
br.setup()
# ---------------------------------------------------------------------------
# macOS setup
# ---------------------------------------------------------------------------
_BH_PRESENT = (
"Audio:\n"
" Devices:\n"
" BlackHole 2ch:\n"
" Manufacturer: Existential Audio\n"
)
_BH_ABSENT = (
"Audio:\n"
" Devices:\n"
" MacBook Pro Microphone:\n"
" Default Input: Yes\n"
)
def test_setup_darwin_returns_blackhole_when_present():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Darwin"), \
patch("plugins.google_meet.audio_bridge.subprocess.check_output",
return_value=_BH_PRESENT) as check:
br = AudioBridge()
info = br.setup()
check.assert_called_once()
argv = check.call_args.args[0]
assert argv[0] == "system_profiler"
assert "SPAudioDataType" in argv
assert info["platform"] == "darwin"
assert info["device_name"] == "BlackHole 2ch"
assert info["write_target"] == "BlackHole 2ch"
assert info["module_ids"] == []
assert info["sample_rate"] == 48000
assert info["channels"] == 2
# teardown is a no-op on darwin (no modules to unload).
with patch("plugins.google_meet.audio_bridge.subprocess.run") as run_mock:
br.teardown()
run_mock.assert_not_called()
def test_setup_darwin_raises_when_blackhole_missing():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Darwin"), \
patch("plugins.google_meet.audio_bridge.subprocess.check_output",
return_value=_BH_ABSENT):
br = AudioBridge()
with pytest.raises(RuntimeError, match="BlackHole"):
br.setup()
# ---------------------------------------------------------------------------
# Windows / unsupported
# ---------------------------------------------------------------------------
def test_setup_windows_raises():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Windows"):
br = AudioBridge()
with pytest.raises(RuntimeError, match="not supported"):
br.setup()
# ---------------------------------------------------------------------------
# chrome_fake_audio_flags
# ---------------------------------------------------------------------------
def test_chrome_fake_audio_flags_linux():
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"):
flags = chrome_fake_audio_flags(
{"platform": "linux", "device_name": "hermes_meet_src"}
)
assert "--use-fake-ui-for-media-stream" in flags
def test_chrome_fake_audio_flags_darwin():
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Darwin"):
flags = chrome_fake_audio_flags(
{"platform": "darwin", "device_name": "BlackHole 2ch"}
)
assert "--use-fake-ui-for-media-stream" in flags
def test_chrome_fake_audio_flags_windows_raises():
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Windows"):
with pytest.raises(RuntimeError):
chrome_fake_audio_flags({"platform": "windows"})
def test_property_access_before_setup_raises():
from plugins.google_meet.audio_bridge import AudioBridge
br = AudioBridge()
with pytest.raises(RuntimeError):
_ = br.device_name
with pytest.raises(RuntimeError):
_ = br.write_target
tests/plugins/test_google_meet_node.py
+674
View File
@@ -0,0 +1,674 @@
"""Tests for the google_meet node primitive.
Covers protocol helpers, the file-backed registry, the server's
token-and-dispatch machinery, a mocked client, and the CLI plumbing.
We never open a real socket — websockets.serve / websockets.sync.client
are fully mocked.
"""
from __future__ import annotations
import argparse
import asyncio
import json
from pathlib import Path
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# protocol.py
# ---------------------------------------------------------------------------
def test_protocol_encode_decode_roundtrip():
from plugins.google_meet.node import protocol
msg = protocol.make_request("ping", "tok", {"x": 1}, req_id="abc")
raw = protocol.encode(msg)
out = protocol.decode(raw)
assert out == msg
assert out["type"] == "ping"
assert out["id"] == "abc"
assert out["token"] == "tok"
assert out["payload"] == {"x": 1}
def test_protocol_make_request_autogenerates_id():
from plugins.google_meet.node import protocol
a = protocol.make_request("ping", "tok", {})
b = protocol.make_request("ping", "tok", {})
assert a["id"] != b["id"]
assert len(a["id"]) >= 16 # uuid4 hex
def test_protocol_make_request_rejects_bad_input():
from plugins.google_meet.node import protocol
with pytest.raises(ValueError):
protocol.make_request("", "tok", {})
with pytest.raises(ValueError):
protocol.make_request("unknown_type", "tok", {})
with pytest.raises(ValueError):
protocol.make_request("ping", "tok", "not a dict") # type: ignore[arg-type]
def test_protocol_decode_raises_on_malformed():
from plugins.google_meet.node import protocol
with pytest.raises(ValueError):
protocol.decode("not json at all")
with pytest.raises(ValueError):
protocol.decode("[]") # list, not object
with pytest.raises(ValueError):
protocol.decode(json.dumps({"id": "x"})) # missing type
with pytest.raises(ValueError):
protocol.decode(json.dumps({"type": "ping"})) # missing id
def test_protocol_validate_request_happy_path():
from plugins.google_meet.node import protocol
msg = protocol.make_request("status", "secret", {})
ok, reason = protocol.validate_request(msg, "secret")
assert ok is True
assert reason == ""
def test_protocol_validate_request_rejects_bad_token():
from plugins.google_meet.node import protocol
msg = protocol.make_request("status", "wrong", {})
ok, reason = protocol.validate_request(msg, "right")
assert ok is False
assert "token" in reason.lower()
def test_protocol_validate_request_rejects_unknown_type():
from plugins.google_meet.node import protocol
raw = {"type": "nope", "id": "1", "token": "t", "payload": {}}
ok, reason = protocol.validate_request(raw, "t")
assert ok is False
assert "unknown" in reason.lower()
def test_protocol_validate_request_rejects_missing_id():
from plugins.google_meet.node import protocol
raw = {"type": "ping", "token": "t", "payload": {}}
ok, reason = protocol.validate_request(raw, "t")
assert ok is False
assert "id" in reason.lower()
def test_protocol_validate_request_rejects_non_dict_payload():
from plugins.google_meet.node import protocol
raw = {"type": "ping", "id": "1", "token": "t", "payload": "oops"}
ok, reason = protocol.validate_request(raw, "t")
assert ok is False
def test_protocol_error_envelope_shape():
from plugins.google_meet.node import protocol
err = protocol.make_error("abc", "nope")
assert err == {"type": "error", "id": "abc", "error": "nope"}
# ---------------------------------------------------------------------------
# registry.py
# ---------------------------------------------------------------------------
def test_registry_add_get_roundtrip_persists(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
p = tmp_path / "nodes.json"
r = NodeRegistry(path=p)
r.add("mac", "ws://mac.local:18789", "deadbeef")
# Second instance sees it.
r2 = NodeRegistry(path=p)
entry = r2.get("mac")
assert entry is not None
assert entry["name"] == "mac"
assert entry["url"] == "ws://mac.local:18789"
assert entry["token"] == "deadbeef"
assert "added_at" in entry
def test_registry_get_returns_none_when_missing(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
assert r.get("ghost") is None
def test_registry_remove(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("a", "ws://a", "t")
assert r.remove("a") is True
assert r.get("a") is None
assert r.remove("a") is False # idempotent
def test_registry_list_all_sorted(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("zeta", "ws://z", "t1")
r.add("alpha", "ws://a", "t2")
names = [n["name"] for n in r.list_all()]
assert names == ["alpha", "zeta"]
def test_registry_resolve_auto_picks_single(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("mac", "ws://mac", "t")
picked = r.resolve(None)
assert picked is not None
assert picked["name"] == "mac"
def test_registry_resolve_ambiguous_returns_none(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("a", "ws://a", "t")
r.add("b", "ws://b", "t")
assert r.resolve(None) is None
def test_registry_resolve_empty_returns_none(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
assert r.resolve(None) is None
def test_registry_resolve_by_name(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("a", "ws://a", "t")
r.add("b", "ws://b", "t")
picked = r.resolve("b")
assert picked is not None
assert picked["name"] == "b"
assert r.resolve("ghost") is None
def test_registry_defaults_to_hermes_home(tmp_path, monkeypatch):
from plugins.google_meet.node.registry import NodeRegistry
# _isolate_home already set HERMES_HOME to tmp_path/.hermes; the
# registry default path must live inside that tree.
r = NodeRegistry()
r.add("x", "ws://x", "t")
expected = Path(tmp_path) / ".hermes" / "workspace" / "meetings" / "nodes.json"
assert expected.is_file()
# ---------------------------------------------------------------------------
# server.py — token + dispatch
# ---------------------------------------------------------------------------
def test_server_ensure_token_generates_and_persists(tmp_path):
from plugins.google_meet.node.server import NodeServer
p = tmp_path / "tok.json"
s1 = NodeServer(token_path=p)
t1 = s1.ensure_token()
assert isinstance(t1, str) and len(t1) == 32
# Reuse on a fresh instance.
s2 = NodeServer(token_path=p)
t2 = s2.ensure_token()
assert t1 == t2
data = json.loads(p.read_text(encoding="utf-8"))
assert data["token"] == t1
assert "generated_at" in data
def test_server_get_token_is_idempotent(tmp_path):
from plugins.google_meet.node.server import NodeServer
s = NodeServer(token_path=tmp_path / "t.json")
assert s.get_token() == s.get_token()
def _run(coro):
return asyncio.new_event_loop().run_until_complete(coro) if False else asyncio.run(coro)
def test_server_handle_request_rejects_bad_token(tmp_path):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
s = NodeServer(token_path=tmp_path / "t.json")
s.ensure_token()
bad = protocol.make_request("ping", "not-the-token", {})
resp = asyncio.run(s._handle_request(bad))
assert resp["type"] == "error"
assert "token" in resp["error"].lower()
def test_server_handle_request_ping(tmp_path):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
s = NodeServer(token_path=tmp_path / "t.json", display_name="node-x")
tok = s.ensure_token()
req = protocol.make_request("ping", tok, {})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "pong"
assert resp["id"] == req["id"]
assert resp["payload"]["display_name"] == "node-x"
def test_server_handle_request_status_dispatches_to_pm(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
monkeypatch.setattr(pm, "status",
lambda: {"ok": True, "alive": True, "meetingId": "abc"})
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("status", tok, {})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["id"] == req["id"]
assert resp["payload"] == {"ok": True, "alive": True, "meetingId": "abc"}
def test_server_handle_request_start_bot_dispatches(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
captured = {}
def fake_start(**kwargs):
captured.update(kwargs)
return {"ok": True, "pid": 42, "meeting_id": "abc-defg-hij"}
monkeypatch.setattr(pm, "start", fake_start)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("start_bot", tok, {
"url": "https://meet.google.com/abc-defg-hij",
"guest_name": "Bot",
"duration": "30m",
})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["ok"] is True
assert captured["url"] == "https://meet.google.com/abc-defg-hij"
assert captured["guest_name"] == "Bot"
assert captured["duration"] == "30m"
def test_server_handle_request_start_bot_missing_url(tmp_path):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("start_bot", tok, {"guest_name": "x"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "error"
assert "url" in resp["error"]
def test_server_handle_request_stop_dispatches(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
got = {}
def fake_stop(*, reason="requested"):
got["reason"] = reason
return {"ok": True, "reason": reason}
monkeypatch.setattr(pm, "stop", fake_stop)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("stop", tok, {"reason": "user-cancel"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert got["reason"] == "user-cancel"
def test_server_handle_request_transcript(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
got = {}
def fake_transcript(last=None):
got["last"] = last
return {"ok": True, "lines": ["a", "b"], "total": 2}
monkeypatch.setattr(pm, "transcript", fake_transcript)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("transcript", tok, {"last": 5})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["lines"] == ["a", "b"]
assert got["last"] == 5
def test_server_handle_request_say_enqueues_when_active(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
out = tmp_path / "meet-out"
out.mkdir()
monkeypatch.setattr(pm, "_read_active",
lambda: {"pid": 1, "meeting_id": "m", "out_dir": str(out)})
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("say", tok, {"text": "hello"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["ok"] is True
assert resp["payload"]["enqueued"] is True
q = (out / "say_queue.jsonl").read_text(encoding="utf-8").strip().splitlines()
assert len(q) == 1
assert json.loads(q[0])["text"] == "hello"
def test_server_handle_request_say_without_active_still_ok(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
monkeypatch.setattr(pm, "_read_active", lambda: None)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("say", tok, {"text": "hi"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["ok"] is True
assert resp["payload"]["enqueued"] is False
def test_server_handle_request_wraps_pm_exceptions(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
def boom():
raise ValueError("kaboom")
monkeypatch.setattr(pm, "status", boom)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("status", tok, {})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "error"
assert "kaboom" in resp["error"]
# ---------------------------------------------------------------------------
# client.py
# ---------------------------------------------------------------------------
class _FakeWS:
"""Minimal context-manager stand-in for websockets.sync.client.connect."""
def __init__(self, reply_builder):
self._reply_builder = reply_builder
self.sent = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def send(self, raw):
self.sent.append(raw)
def recv(self, timeout=None):
return self._reply_builder(self.sent[-1])
def _install_fake_ws(monkeypatch, reply_builder):
fake_ws_holder = {}
def _connect(url, **kwargs):
ws = _FakeWS(reply_builder)
fake_ws_holder["ws"] = ws
fake_ws_holder["url"] = url
fake_ws_holder["kwargs"] = kwargs
return ws
# Patch the concrete import site inside client._rpc
import websockets.sync.client as wsc # type: ignore
monkeypatch.setattr(wsc, "connect", _connect)
return fake_ws_holder
def test_client_rpc_sends_correct_envelope_and_parses_response(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
def reply(raw_out):
req = protocol.decode(raw_out)
return protocol.encode(protocol.make_response(req["id"], {"ok": True, "echo": req["type"]}))
holder = _install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://remote:1", "tok123")
out = c._rpc("ping", {"hello": 1})
assert out == {"ok": True, "echo": "ping"}
sent = json.loads(holder["ws"].sent[0])
assert sent["type"] == "ping"
assert sent["token"] == "tok123"
assert sent["payload"] == {"hello": 1}
assert sent["id"] # non-empty
assert holder["url"] == "ws://remote:1"
def test_client_rpc_raises_on_error_envelope(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
def reply(raw_out):
req = protocol.decode(raw_out)
return protocol.encode(protocol.make_error(req["id"], "nope"))
_install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://x", "t")
with pytest.raises(RuntimeError, match="nope"):
c._rpc("ping", {})
def test_client_rpc_raises_on_id_mismatch(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
def reply(raw_out):
return protocol.encode(protocol.make_response("different-id", {"ok": True}))
_install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://x", "t")
with pytest.raises(RuntimeError, match="mismatch"):
c._rpc("ping", {})
def test_client_convenience_methods_hit_correct_types(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
seen = []
def reply(raw_out):
req = protocol.decode(raw_out)
seen.append((req["type"], req["payload"]))
return protocol.encode(protocol.make_response(req["id"], {"ok": True}))
_install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://x", "t")
c.start_bot("https://meet.google.com/a-b-c", guest_name="G", duration="10m")
c.stop()
c.status()
c.transcript(last=3)
c.say("hi")
c.ping()
types = [t for t, _ in seen]
assert types == ["start_bot", "stop", "status", "transcript", "say", "ping"]
# Check specific payload routing
assert seen[0][1]["url"] == "https://meet.google.com/a-b-c"
assert seen[0][1]["guest_name"] == "G"
assert seen[0][1]["duration"] == "10m"
assert seen[3][1]["last"] == 3
assert seen[4][1]["text"] == "hi"
def test_client_init_rejects_bad_args():
from plugins.google_meet.node.client import NodeClient
with pytest.raises(ValueError):
NodeClient("", "t")
with pytest.raises(ValueError):
NodeClient("ws://x", "")
# ---------------------------------------------------------------------------
# cli.py
# ---------------------------------------------------------------------------
def _build_parser():
from plugins.google_meet.node.cli import register_cli
parser = argparse.ArgumentParser(prog="meet-node-test")
register_cli(parser)
return parser
def test_cli_approve_list_remove(capsys):
from plugins.google_meet.node.registry import NodeRegistry
p = _build_parser()
args = p.parse_args(["approve", "mac", "ws://mac:1", "tok"])
rc = args.func(args)
assert rc == 0
assert NodeRegistry().get("mac") is not None
args = p.parse_args(["list"])
rc = args.func(args)
assert rc == 0
out = capsys.readouterr().out
assert "mac" in out
assert "ws://mac:1" in out
args = p.parse_args(["remove", "mac"])
rc = args.func(args)
assert rc == 0
assert NodeRegistry().get("mac") is None
def test_cli_list_empty(capsys):
p = _build_parser()
args = p.parse_args(["list"])
rc = args.func(args)
assert rc == 0
assert "no nodes" in capsys.readouterr().out
def test_cli_remove_missing_returns_nonzero():
p = _build_parser()
args = p.parse_args(["remove", "ghost"])
rc = args.func(args)
assert rc == 1
def test_cli_status_pings_via_node_client(capsys, monkeypatch):
from plugins.google_meet.node.registry import NodeRegistry
from plugins.google_meet.node import cli as node_cli
NodeRegistry().add("mac", "ws://mac:1", "tok")
class _FakeClient:
def __init__(self, url, token):
assert url == "ws://mac:1"
assert token == "tok"
def ping(self):
return {"type": "pong", "display_name": "hermes-meet-node"}
monkeypatch.setattr(node_cli, "NodeClient", _FakeClient)
p = _build_parser()
args = p.parse_args(["status", "mac"])
rc = args.func(args)
assert rc == 0
out = capsys.readouterr().out.strip()
data = json.loads(out)
assert data["ok"] is True
assert data["node"] == "mac"
def test_cli_status_unknown_node_fails(capsys):
p = _build_parser()
args = p.parse_args(["status", "ghost"])
rc = args.func(args)
assert rc == 1
def test_cli_status_reports_client_error(capsys, monkeypatch):
from plugins.google_meet.node.registry import NodeRegistry
from plugins.google_meet.node import cli as node_cli
NodeRegistry().add("mac", "ws://mac:1", "tok")
class _FakeClient:
def __init__(self, url, token):
pass
def ping(self):
raise RuntimeError("connection refused")
monkeypatch.setattr(node_cli, "NodeClient", _FakeClient)
p = _build_parser()
args = p.parse_args(["status", "mac"])
rc = args.func(args)
assert rc == 1
data = json.loads(capsys.readouterr().out.strip())
assert data["ok"] is False
assert "connection refused" in data["error"]
tests/plugins/test_google_meet_plugin.py
+813
View File
@@ -0,0 +1,813 @@
"""Tests for the google_meet plugin.
Covers the safety-gated pieces that don't require Playwright:
* URL regex — only ``https://meet.google.com/`` URLs pass
* Meeting-id extraction from Meet URLs
* Status / transcript writes round-trip through the file-backed state
* Tool handlers return well-formed JSON under all branches
* Process manager refuses unsafe URLs and clears stale state cleanly
* ``_on_session_end`` hook is defensive (no-ops when no bot active)
Does NOT spawn a real Chromium — we mock ``subprocess.Popen`` where needed.
"""
from __future__ import annotations
import json
import os
import signal
from pathlib import Path
from unittest.mock import patch
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# URL safety gate
# ---------------------------------------------------------------------------
def test_is_safe_meet_url_accepts_standard_meet_codes():
from plugins.google_meet.meet_bot import _is_safe_meet_url
assert _is_safe_meet_url("https://meet.google.com/abc-defg-hij")
assert _is_safe_meet_url("https://meet.google.com/abc-defg-hij?pli=1")
assert _is_safe_meet_url("https://meet.google.com/new")
assert _is_safe_meet_url("https://meet.google.com/lookup/ABC123")
def test_is_safe_meet_url_rejects_non_meet_urls():
from plugins.google_meet.meet_bot import _is_safe_meet_url
# wrong host
assert not _is_safe_meet_url("https://evil.example.com/abc-defg-hij")
# wrong scheme
assert not _is_safe_meet_url("http://meet.google.com/abc-defg-hij")
# malformed code
assert not _is_safe_meet_url("https://meet.google.com/not-a-meet-code")
# subdomain hijack attempts
assert not _is_safe_meet_url("https://meet.google.com.evil.com/abc-defg-hij")
assert not _is_safe_meet_url("https://notmeet.google.com/abc-defg-hij")
# empty / wrong type
assert not _is_safe_meet_url("")
assert not _is_safe_meet_url(None) # type: ignore[arg-type]
assert not _is_safe_meet_url(123) # type: ignore[arg-type]
def test_meeting_id_extraction():
from plugins.google_meet.meet_bot import _meeting_id_from_url
assert _meeting_id_from_url("https://meet.google.com/abc-defg-hij") == "abc-defg-hij"
assert _meeting_id_from_url("https://meet.google.com/abc-defg-hij?pli=1") == "abc-defg-hij"
# fallback for codes we can't parse (e.g. /new before redirect)
fallback = _meeting_id_from_url("https://meet.google.com/new")
assert fallback.startswith("meet-")
# ---------------------------------------------------------------------------
# _BotState — transcript + status file round-trip
# ---------------------------------------------------------------------------
def test_bot_state_dedupes_captions_and_flushes_status(tmp_path):
from plugins.google_meet.meet_bot import _BotState
out = tmp_path / "session"
state = _BotState(out_dir=out, meeting_id="abc-defg-hij",
url="https://meet.google.com/abc-defg-hij")
state.record_caption("Alice", "Hey everyone")
state.record_caption("Alice", "Hey everyone") # dup — ignored
state.record_caption("Bob", "Let's start")
transcript = (out / "transcript.txt").read_text()
assert "Alice: Hey everyone" in transcript
assert "Bob: Let's start" in transcript
# dedup — Alice line appears exactly once
assert transcript.count("Alice: Hey everyone") == 1
status = json.loads((out / "status.json").read_text())
assert status["meetingId"] == "abc-defg-hij"
assert status["transcriptLines"] == 2
assert status["transcriptPath"].endswith("transcript.txt")
def test_bot_state_ignores_blank_text(tmp_path):
from plugins.google_meet.meet_bot import _BotState
state = _BotState(out_dir=tmp_path / "s", meeting_id="x-y-z",
url="https://meet.google.com/x-y-z")
state.record_caption("Alice", "")
state.record_caption("Alice", " ")
state.record_caption("", "text but no speaker")
status = json.loads((tmp_path / "s" / "status.json").read_text())
assert status["transcriptLines"] == 1
# blank-speaker falls back to "Unknown"
assert "Unknown: text but no speaker" in (tmp_path / "s" / "transcript.txt").read_text()
def test_parse_duration():
from plugins.google_meet.meet_bot import _parse_duration
assert _parse_duration("30m") == 30 * 60
assert _parse_duration("2h") == 2 * 3600
assert _parse_duration("90s") == 90
assert _parse_duration("90") == 90
assert _parse_duration("") is None
assert _parse_duration("bogus") is None
# ---------------------------------------------------------------------------
# process_manager — refuses unsafe URLs, manages active pointer
# ---------------------------------------------------------------------------
def test_start_refuses_unsafe_url():
from plugins.google_meet import process_manager as pm
res = pm.start("https://evil.example.com/abc-defg-hij")
assert res["ok"] is False
assert "refusing" in res["error"]
def test_status_reports_no_active_meeting():
from plugins.google_meet import process_manager as pm
assert pm.status() == {"ok": False, "reason": "no active meeting"}
assert pm.transcript() == {"ok": False, "reason": "no active meeting"}
assert pm.stop() == {"ok": False, "reason": "no active meeting"}
def test_start_spawns_subprocess_and_writes_active_pointer(tmp_path):
"""Verify start() wires env vars correctly and records the pid."""
from plugins.google_meet import process_manager as pm
class _FakeProc:
def __init__(self, pid):
self.pid = pid
captured_env = {}
captured_argv = []
def _fake_popen(argv, **kwargs):
captured_argv.extend(argv)
captured_env.update(kwargs.get("env") or {})
return _FakeProc(99999)
with patch.object(pm.subprocess, "Popen", side_effect=_fake_popen):
# Also prevent pid liveness probe from stomping on our real pids
with patch.object(pm, "_pid_alive", return_value=False):
res = pm.start(
"https://meet.google.com/abc-defg-hij",
guest_name="Test Bot",
duration="15m",
)
assert res["ok"] is True
assert res["meeting_id"] == "abc-defg-hij"
assert res["pid"] == 99999
assert captured_env["HERMES_MEET_URL"] == "https://meet.google.com/abc-defg-hij"
assert captured_env["HERMES_MEET_GUEST_NAME"] == "Test Bot"
assert captured_env["HERMES_MEET_DURATION"] == "15m"
# python -m plugins.google_meet.meet_bot
assert any("plugins.google_meet.meet_bot" in a for a in captured_argv)
# .active.json points at the bot
active = pm._read_active()
assert active is not None
assert active["pid"] == 99999
assert active["meeting_id"] == "abc-defg-hij"
def test_transcript_reads_last_n_lines(tmp_path):
from plugins.google_meet import process_manager as pm
meeting_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
meeting_dir.mkdir(parents=True)
(meeting_dir / "transcript.txt").write_text(
"[10:00:00] Alice: one\n"
"[10:00:01] Bob: two\n"
"[10:00:02] Alice: three\n"
)
pm._write_active({
"pid": 0, "meeting_id": "abc-defg-hij",
"out_dir": str(meeting_dir),
"url": "https://meet.google.com/abc-defg-hij",
"started_at": 0,
})
res = pm.transcript(last=2)
assert res["ok"] is True
assert res["total"] == 3
assert len(res["lines"]) == 2
assert res["lines"][-1].endswith("Alice: three")
def test_stop_signals_process_and_clears_pointer(tmp_path):
from plugins.google_meet import process_manager as pm
pm._write_active({
"pid": 11111, "meeting_id": "x-y-z",
"out_dir": str(tmp_path / "x-y-z"),
"url": "https://meet.google.com/x-y-z",
"started_at": 0,
})
alive_seq = iter([True, True, False]) # alive at first, gone after SIGTERM
def _alive(pid):
try:
return next(alive_seq)
except StopIteration:
return False
sent = []
def _kill(pid, sig):
sent.append((pid, sig))
with patch.object(pm, "_pid_alive", side_effect=_alive), \
patch.object(pm.os, "kill", side_effect=_kill), \
patch.object(pm.time, "sleep", lambda _s: None):
res = pm.stop()
assert res["ok"] is True
assert (11111, signal.SIGTERM) in sent
# .active.json cleared
assert pm._read_active() is None
# ---------------------------------------------------------------------------
# Tool handlers — JSON shape + safety gates
# ---------------------------------------------------------------------------
def test_meet_join_handler_missing_url_returns_error():
from plugins.google_meet.tools import handle_meet_join
out = json.loads(handle_meet_join({}))
assert out["success"] is False
assert "url is required" in out["error"]
def test_meet_join_handler_respects_safety_gate():
from plugins.google_meet.tools import handle_meet_join
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=True):
out = json.loads(handle_meet_join({"url": "https://evil.example.com/foo"}))
assert out["success"] is False
assert "refusing" in out["error"]
def test_meet_join_handler_returns_error_when_playwright_missing():
from plugins.google_meet.tools import handle_meet_join
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=False):
out = json.loads(handle_meet_join({"url": "https://meet.google.com/abc-defg-hij"}))
assert out["success"] is False
assert "prerequisites missing" in out["error"]
def test_meet_say_requires_text():
from plugins.google_meet.tools import handle_meet_say
out = json.loads(handle_meet_say({}))
assert out["success"] is False
assert "text is required" in out["error"]
def test_meet_say_no_active_meeting():
from plugins.google_meet.tools import handle_meet_say
out = json.loads(handle_meet_say({"text": "hello everyone"}))
assert out["success"] is False
# Falls through to pm.enqueue_say which reports no active meeting.
assert "no active meeting" in out.get("reason", "")
def test_meet_status_and_transcript_no_active():
from plugins.google_meet.tools import handle_meet_status, handle_meet_transcript
assert json.loads(handle_meet_status({}))["success"] is False
assert json.loads(handle_meet_transcript({}))["success"] is False
def test_meet_leave_no_active():
from plugins.google_meet.tools import handle_meet_leave
out = json.loads(handle_meet_leave({}))
assert out["success"] is False
# ---------------------------------------------------------------------------
# _on_session_end — defensive cleanup
# ---------------------------------------------------------------------------
def test_on_session_end_noop_when_nothing_active():
from plugins.google_meet import _on_session_end
# Should not raise and should not call stop().
with patch("plugins.google_meet.pm.stop") as stop_mock:
_on_session_end()
stop_mock.assert_not_called()
def test_on_session_end_stops_live_bot():
from plugins.google_meet import _on_session_end
from plugins.google_meet import pm
with patch.object(pm, "status", return_value={"ok": True, "alive": True}), \
patch.object(pm, "stop") as stop_mock:
_on_session_end()
stop_mock.assert_called_once()
# ---------------------------------------------------------------------------
# Plugin register() — platform gating + tool registration
# ---------------------------------------------------------------------------
def test_register_refuses_on_windows():
import plugins.google_meet as plugin
calls = {"tools": [], "cli": [], "hooks": []}
class _Ctx:
def register_tool(self, **kw): calls["tools"].append(kw["name"])
def register_cli_command(self, **kw): calls["cli"].append(kw["name"])
def register_hook(self, name, fn): calls["hooks"].append(name)
with patch.object(plugin.platform, "system", return_value="Windows"):
plugin.register(_Ctx())
assert calls == {"tools": [], "cli": [], "hooks": []}
def test_register_wires_tools_cli_and_hook_on_linux():
import plugins.google_meet as plugin
calls = {"tools": [], "cli": [], "hooks": []}
class _Ctx:
def register_tool(self, **kw): calls["tools"].append(kw["name"])
def register_cli_command(self, **kw): calls["cli"].append(kw["name"])
def register_hook(self, name, fn): calls["hooks"].append(name)
with patch.object(plugin.platform, "system", return_value="Linux"):
plugin.register(_Ctx())
assert set(calls["tools"]) == {
"meet_join", "meet_status", "meet_transcript", "meet_leave", "meet_say",
}
assert calls["cli"] == ["meet"]
assert calls["hooks"] == ["on_session_end"]
# ---------------------------------------------------------------------------
# v2: process_manager.enqueue_say + realtime-mode passthrough
# ---------------------------------------------------------------------------
def test_enqueue_say_requires_text():
from plugins.google_meet import process_manager as pm
assert pm.enqueue_say("")["ok"] is False
assert pm.enqueue_say(" ")["ok"] is False
def test_enqueue_say_no_active_meeting():
from plugins.google_meet import process_manager as pm
res = pm.enqueue_say("hi team")
assert res["ok"] is False
assert "no active meeting" in res["reason"]
def test_enqueue_say_rejects_transcribe_mode(tmp_path):
from plugins.google_meet import process_manager as pm
out_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
out_dir.mkdir(parents=True)
pm._write_active({
"pid": 0, "meeting_id": "abc-defg-hij",
"out_dir": str(out_dir), "url": "https://meet.google.com/abc-defg-hij",
"started_at": 0, "mode": "transcribe",
})
res = pm.enqueue_say("hi team")
assert res["ok"] is False
assert "transcribe mode" in res["reason"]
def test_enqueue_say_writes_jsonl_in_realtime_mode():
from plugins.google_meet import process_manager as pm
out_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
out_dir.mkdir(parents=True)
pm._write_active({
"pid": 0, "meeting_id": "abc-defg-hij",
"out_dir": str(out_dir), "url": "https://meet.google.com/abc-defg-hij",
"started_at": 0, "mode": "realtime",
})
res = pm.enqueue_say("hello everyone")
assert res["ok"] is True
assert "enqueued_id" in res
queue = out_dir / "say_queue.jsonl"
assert queue.is_file()
lines = [json.loads(ln) for ln in queue.read_text().splitlines() if ln.strip()]
assert len(lines) == 1
assert lines[0]["text"] == "hello everyone"
def test_start_passes_mode_into_active_record():
from plugins.google_meet import process_manager as pm
class _FakeProc:
def __init__(self, pid): self.pid = pid
with patch.object(pm.subprocess, "Popen", return_value=_FakeProc(12345)), \
patch.object(pm, "_pid_alive", return_value=False):
res = pm.start(
"https://meet.google.com/abc-defg-hij",
mode="realtime",
)
assert res["ok"] is True
assert res["mode"] == "realtime"
assert pm._read_active()["mode"] == "realtime"
def test_start_realtime_env_vars_threaded_through():
from plugins.google_meet import process_manager as pm
class _FakeProc:
def __init__(self, pid): self.pid = pid
captured_env = {}
def _fake_popen(argv, **kwargs):
captured_env.update(kwargs.get("env") or {})
return _FakeProc(11111)
with patch.object(pm.subprocess, "Popen", side_effect=_fake_popen), \
patch.object(pm, "_pid_alive", return_value=False):
pm.start(
"https://meet.google.com/abc-defg-hij",
mode="realtime",
realtime_model="gpt-realtime",
realtime_voice="alloy",
realtime_instructions="Be brief.",
realtime_api_key="sk-test",
)
assert captured_env["HERMES_MEET_MODE"] == "realtime"
assert captured_env["HERMES_MEET_REALTIME_MODEL"] == "gpt-realtime"
assert captured_env["HERMES_MEET_REALTIME_VOICE"] == "alloy"
assert captured_env["HERMES_MEET_REALTIME_INSTRUCTIONS"] == "Be brief."
assert captured_env["HERMES_MEET_REALTIME_KEY"] == "sk-test"
def test_meet_join_accepts_realtime_mode():
from plugins.google_meet.tools import handle_meet_join
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=True), \
patch("plugins.google_meet.tools.pm.start", return_value={"ok": True, "meeting_id": "x-y-z"}) as start_mock:
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"mode": "realtime",
}))
assert out["success"] is True
assert start_mock.call_args.kwargs["mode"] == "realtime"
def test_meet_join_rejects_bad_mode():
from plugins.google_meet.tools import handle_meet_join
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"mode": "bogus",
}))
assert out["success"] is False
assert "mode must be" in out["error"]
# ---------------------------------------------------------------------------
# v3: NodeClient routing from tool handlers
# ---------------------------------------------------------------------------
def test_meet_join_unknown_node_returns_clear_error():
from plugins.google_meet.tools import handle_meet_join
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "my-mac",
}))
assert out["success"] is False
assert "no registered meet node" in out["error"]
def test_meet_join_routes_to_registered_node():
from plugins.google_meet.tools import handle_meet_join
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("my-mac", "ws://1.2.3.4:18789", "tok")
with patch("plugins.google_meet.node.client.NodeClient.start_bot",
return_value={"ok": True, "meeting_id": "a-b-c"}) as call_mock:
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "my-mac",
"mode": "realtime",
}))
assert out["success"] is True
assert out["node"] == "my-mac"
assert call_mock.call_args.kwargs["mode"] == "realtime"
def test_meet_say_routes_to_node():
from plugins.google_meet.tools import handle_meet_say
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("my-mac", "ws://1.2.3.4:18789", "tok")
with patch("plugins.google_meet.node.client.NodeClient.say",
return_value={"ok": True, "enqueued_id": "abc"}) as call_mock:
out = json.loads(handle_meet_say({"text": "hello", "node": "my-mac"}))
assert out["success"] is True
assert out["node"] == "my-mac"
call_mock.assert_called_once_with("hello")
def test_meet_join_auto_node_selects_sole_registered():
from plugins.google_meet.tools import handle_meet_join
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("only-one", "ws://1.2.3.4:18789", "tok")
with patch("plugins.google_meet.node.client.NodeClient.start_bot",
return_value={"ok": True}) as call_mock:
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "auto",
}))
assert out["success"] is True
assert out["node"] == "only-one"
assert call_mock.called
def test_meet_join_auto_node_ambiguous_returns_error():
from plugins.google_meet.tools import handle_meet_join
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("a", "ws://1.2.3.4:18789", "tok")
reg.add("b", "ws://5.6.7.8:18789", "tok")
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "auto",
}))
assert out["success"] is False
assert "no registered meet node" in out["error"]
def test_cli_register_includes_node_subcommand():
"""`hermes meet` argparse tree includes the node subtree."""
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
# Parse a known-good node invocation to prove the subtree is wired.
ns = parser.parse_args(["node", "list"])
assert ns.meet_command == "node"
assert ns.node_cmd == "list"
def test_cli_join_accepts_mode_and_node_flags():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args([
"join", "https://meet.google.com/abc-defg-hij",
"--mode", "realtime", "--node", "my-mac",
])
assert ns.mode == "realtime"
assert ns.node == "my-mac"
def test_cli_say_subcommand_exists():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args(["say", "hello team", "--node", "my-mac"])
assert ns.text == "hello team"
assert ns.node == "my-mac"
# ---------------------------------------------------------------------------
# v2.1: new _BotState fields + status dict shape
# ---------------------------------------------------------------------------
def test_bot_state_exposes_v2_telemetry_fields(tmp_path):
from plugins.google_meet.meet_bot import _BotState
state = _BotState(out_dir=tmp_path / "s", meeting_id="x-y-z",
url="https://meet.google.com/x-y-z")
# Defaults for the new fields.
status = json.loads((tmp_path / "s" / "status.json").read_text())
for key in (
"realtime", "realtimeReady", "realtimeDevice",
"audioBytesOut", "lastAudioOutAt", "lastBargeInAt",
"joinAttemptedAt", "leaveReason",
):
assert key in status, f"missing v2 telemetry key: {key}"
assert status["realtime"] is False
assert status["realtimeReady"] is False
assert status["audioBytesOut"] == 0
# Setting them flushes them.
state.set(realtime=True, realtime_ready=True, audio_bytes_out=1024,
leave_reason="lobby_timeout")
status = json.loads((tmp_path / "s" / "status.json").read_text())
assert status["realtime"] is True
assert status["realtimeReady"] is True
assert status["audioBytesOut"] == 1024
assert status["leaveReason"] == "lobby_timeout"
# ---------------------------------------------------------------------------
# Admission detection + barge-in helper
# ---------------------------------------------------------------------------
def test_looks_like_human_speaker():
from plugins.google_meet.meet_bot import _looks_like_human_speaker
# Blank, "unknown", "you", and the bot's own name → not human (no barge-in)
for s in ("", " ", "Unknown", "unknown", "You", "you", "Hermes Agent", "hermes agent"):
assert not _looks_like_human_speaker(s, "Hermes Agent"), f"{s!r} should NOT be human"
# Real names → human (barge-in)
for s in ("Alice", "Bob Lee", "@teknium"):
assert _looks_like_human_speaker(s, "Hermes Agent"), f"{s!r} SHOULD be human"
def test_detect_admission_returns_false_on_error():
from plugins.google_meet.meet_bot import _detect_admission
class _FakePage:
def evaluate(self, _js): raise RuntimeError("boom")
assert _detect_admission(_FakePage()) is False
def test_detect_admission_true_when_probe_returns_true():
from plugins.google_meet.meet_bot import _detect_admission
class _FakePage:
def evaluate(self, _js): return True
assert _detect_admission(_FakePage()) is True
def test_detect_denied_returns_false_on_error():
from plugins.google_meet.meet_bot import _detect_denied
class _FakePage:
def evaluate(self, _js): raise RuntimeError("boom")
assert _detect_denied(_FakePage()) is False
# ---------------------------------------------------------------------------
# Realtime session counters + cancel_response (barge-in)
# ---------------------------------------------------------------------------
def test_realtime_session_cancel_response_when_disconnected():
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
# No _ws yet — cancel should no-op and return False.
assert sess.cancel_response() is False
def test_realtime_session_cancel_response_sends_cancel_frame():
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
sent = []
class _FakeWs:
def send(self, msg): sent.append(msg)
sess._ws = _FakeWs()
assert sess.cancel_response() is True
assert len(sent) == 1
import json as _j
envelope = _j.loads(sent[0])
assert envelope == {"type": "response.cancel"}
def test_realtime_session_counters_initialized():
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
assert sess.audio_bytes_out == 0
assert sess.last_audio_out_at is None
# ---------------------------------------------------------------------------
# hermes meet install CLI
# ---------------------------------------------------------------------------
def test_cli_install_subcommand_is_registered():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args(["install"])
assert ns.meet_command == "install"
assert ns.realtime is False
assert ns.yes is False
def test_cli_install_flags_parse():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args(["install", "--realtime", "--yes"])
assert ns.realtime is True
assert ns.yes is True
def test_cmd_install_refuses_windows(capsys):
from plugins.google_meet.cli import _cmd_install
with patch("plugins.google_meet.cli.platform" if False else "platform.system",
return_value="Windows"):
rc = _cmd_install(realtime=False, assume_yes=True)
assert rc == 1
out = capsys.readouterr().out
assert "Windows" in out
def test_cmd_install_runs_pip_and_playwright(capsys):
"""End-to-end wiring: pip + playwright install invoked, returncodes handled."""
from plugins.google_meet.cli import _cmd_install
calls = []
class _FakeRes:
def __init__(self, rc=0): self.returncode = rc
def _fake_run(argv, **kwargs):
calls.append(list(argv))
return _FakeRes(0)
with patch("platform.system", return_value="Linux"), \
patch("subprocess.run", side_effect=_fake_run), \
patch("shutil.which", return_value="/usr/bin/paplay"):
rc = _cmd_install(realtime=False, assume_yes=True)
assert rc == 0
# First invocation: pip install
pip_cmds = [c for c in calls if len(c) > 2 and c[1:4] == ["-m", "pip", "install"]]
assert pip_cmds, f"no pip install run: {calls}"
assert "playwright" in pip_cmds[0]
assert "websockets" in pip_cmds[0]
# Second: playwright install chromium
pw_cmds = [c for c in calls if len(c) > 2 and c[1:4] == ["-m", "playwright", "install"]]
assert pw_cmds, f"no playwright install run: {calls}"
assert "chromium" in pw_cmds[0]
def test_cmd_install_realtime_skips_when_deps_present(capsys):
"""When paplay + pactl are already on PATH, no sudo call happens."""
from plugins.google_meet.cli import _cmd_install
calls = []
class _FakeRes:
def __init__(self, rc=0): self.returncode = rc
def _fake_run(argv, **kwargs):
calls.append(list(argv))
return _FakeRes(0)
with patch("platform.system", return_value="Linux"), \
patch("subprocess.run", side_effect=_fake_run), \
patch("shutil.which", return_value="/usr/bin/paplay"):
rc = _cmd_install(realtime=True, assume_yes=True)
assert rc == 0
# No sudo apt-get call — paplay was already on PATH.
sudo_calls = [c for c in calls if c and c[0] == "sudo"]
assert sudo_calls == [], f"unexpected sudo invocation: {sudo_calls}"
out = capsys.readouterr().out
assert "already installed" in out
tests/plugins/test_google_meet_realtime.py
+290
View File
@@ -0,0 +1,290 @@
"""Tests for plugins.google_meet.realtime.openai_client (v2).
Uses a scripted fake WebSocket — no network, no API key required.
"""
from __future__ import annotations
import base64
import json
import sys
import types
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# Fake WebSocket
# ---------------------------------------------------------------------------
class _FakeWS:
"""Scripted WS: send() records frames, recv() pops a queue."""
def __init__(self, recv_frames: list):
self.sent: list[dict] = []
self._recv_q: list = list(recv_frames)
self.closed = False
def send(self, payload):
# Always accept str payloads — client encodes JSON with json.dumps.
if isinstance(payload, (bytes, bytearray)):
payload = payload.decode()
self.sent.append(json.loads(payload))
def recv(self, timeout=None): # noqa: ARG002
if not self._recv_q:
raise RuntimeError("fake ws: no more frames")
frame = self._recv_q.pop(0)
if isinstance(frame, dict):
return json.dumps(frame)
return frame
def close(self):
self.closed = True
def _install_fake_websockets(monkeypatch, fake_ws):
"""Install a fake ``websockets.sync.client`` module in sys.modules."""
mod_websockets = types.ModuleType("websockets")
mod_sync = types.ModuleType("websockets.sync")
mod_sync_client = types.ModuleType("websockets.sync.client")
captured = {"url": None, "headers": None, "kwargs": None}
def _connect(url, **kwargs):
captured["url"] = url
captured["kwargs"] = kwargs
captured["headers"] = (
kwargs.get("additional_headers") or kwargs.get("extra_headers")
)
return fake_ws
mod_sync_client.connect = _connect
mod_sync.client = mod_sync_client
mod_websockets.sync = mod_sync
monkeypatch.setitem(sys.modules, "websockets", mod_websockets)
monkeypatch.setitem(sys.modules, "websockets.sync", mod_sync)
monkeypatch.setitem(sys.modules, "websockets.sync.client", mod_sync_client)
return captured
# ---------------------------------------------------------------------------
# connect()
# ---------------------------------------------------------------------------
def test_connect_sends_session_update_with_voice_and_instructions(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
ws = _FakeWS(recv_frames=[])
captured = _install_fake_websockets(monkeypatch, ws)
sess = RealtimeSession(
api_key="sk-test",
model="gpt-realtime",
voice="verse",
instructions="Be brief.",
)
sess.connect()
# Auth + beta headers set.
assert captured["url"].startswith("wss://api.openai.com/v1/realtime")
assert "model=gpt-realtime" in captured["url"]
headers = captured["headers"] or []
hdict = dict(headers)
assert hdict.get("Authorization") == "Bearer sk-test"
assert hdict.get("OpenAI-Beta") == "realtime=v1"
# First frame sent must be session.update with the right shape.
assert len(ws.sent) == 1
update = ws.sent[0]
assert update["type"] == "session.update"
s = update["session"]
assert s["voice"] == "verse"
assert s["instructions"] == "Be brief."
assert set(s["modalities"]) == {"audio", "text"}
assert s["output_audio_format"] == "pcm16"
assert s["input_audio_format"] == "pcm16"
# ---------------------------------------------------------------------------
# speak()
# ---------------------------------------------------------------------------
def test_speak_sends_create_and_response_and_writes_audio(monkeypatch, tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSession
audio_bytes = b"\x01\x02\x03\x04PCM!"
b64 = base64.b64encode(audio_bytes).decode()
recv_frames = [
{"type": "response.created"},
{"type": "response.audio.delta", "delta": b64},
{"type": "response.audio.delta", "delta": base64.b64encode(b"more").decode()},
{"type": "response.done"},
]
ws = _FakeWS(recv_frames=recv_frames)
_install_fake_websockets(monkeypatch, ws)
sink = tmp_path / "out.pcm"
sess = RealtimeSession(api_key="sk-test", audio_sink_path=sink)
sess.connect()
result = sess.speak("Hello everyone.")
# Frames sent after session.update: conversation.item.create then response.create.
types_sent = [f["type"] for f in ws.sent]
assert types_sent == ["session.update", "conversation.item.create", "response.create"]
item = ws.sent[1]["item"]
assert item["role"] == "user"
assert item["content"][0]["type"] == "input_text"
assert item["content"][0]["text"] == "Hello everyone."
resp = ws.sent[2]["response"]
assert resp["modalities"] == ["audio"]
# Audio file got decoded + appended bytes.
data = sink.read_bytes()
assert data == audio_bytes + b"more"
assert result["ok"] is True
assert result["bytes_written"] == len(audio_bytes) + len(b"more")
assert result["duration_ms"] >= 0.0
def test_speak_raises_on_error_frame(monkeypatch, tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSession
ws = _FakeWS(recv_frames=[
{"type": "response.created"},
{"type": "error", "error": {"message": "bad juju"}},
])
_install_fake_websockets(monkeypatch, ws)
sess = RealtimeSession(api_key="sk-test", audio_sink_path=tmp_path / "o.pcm")
sess.connect()
with pytest.raises(RuntimeError, match="bad juju"):
sess.speak("hi")
def test_speak_without_connect_raises(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test")
with pytest.raises(RuntimeError, match="connect"):
sess.speak("hi")
def test_close_is_idempotent_and_closes_ws(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
ws = _FakeWS(recv_frames=[])
_install_fake_websockets(monkeypatch, ws)
sess = RealtimeSession(api_key="sk-test")
sess.connect()
sess.close()
assert ws.closed is True
# Second close is a no-op.
sess.close()
# ---------------------------------------------------------------------------
# websockets dependency missing
# ---------------------------------------------------------------------------
def test_connect_raises_clean_error_when_websockets_missing(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
# Make `import websockets.sync.client` fail.
monkeypatch.setitem(sys.modules, "websockets", None)
monkeypatch.setitem(sys.modules, "websockets.sync", None)
monkeypatch.setitem(sys.modules, "websockets.sync.client", None)
sess = RealtimeSession(api_key="sk-test")
with pytest.raises(RuntimeError, match="pip install websockets"):
sess.connect()
# ---------------------------------------------------------------------------
# RealtimeSpeaker
# ---------------------------------------------------------------------------
class _StubSession:
def __init__(self):
self.spoken: list[str] = []
def speak(self, text, timeout=30.0): # noqa: ARG002
self.spoken.append(text)
return {"ok": True, "bytes_written": len(text), "duration_ms": 1.0}
def test_speaker_run_until_stopped_processes_queue(tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
queue = tmp_path / "queue.jsonl"
processed = tmp_path / "processed.jsonl"
queue.write_text(
json.dumps({"id": "a", "text": "hello one"}) + "\n"
+ json.dumps({"id": "b", "text": "hello two"}) + "\n"
)
stub = _StubSession()
speaker = RealtimeSpeaker(stub, queue_path=queue, processed_path=processed)
# Stop once the queue is empty.
def _stop():
return queue.exists() and queue.read_text().strip() == ""
speaker.run_until_stopped(_stop, poll_interval=0.01)
assert stub.spoken == ["hello one", "hello two"]
# Processed file has both entries, in order.
lines = [json.loads(l) for l in processed.read_text().splitlines() if l.strip()]
assert [l["id"] for l in lines] == ["a", "b"]
assert all(l["result"]["ok"] for l in lines)
# Queue is empty (possibly empty string) after processing.
assert queue.read_text().strip() == ""
def test_speaker_exits_immediately_when_stop_fn_true(tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
queue = tmp_path / "q.jsonl"
queue.write_text(json.dumps({"id": "x", "text": "never spoken"}) + "\n")
stub = _StubSession()
speaker = RealtimeSpeaker(stub, queue_path=queue)
speaker.run_until_stopped(lambda: True, poll_interval=0.01)
assert stub.spoken == []
def test_speaker_drops_line_without_processed_path_when_none(tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
queue = tmp_path / "q.jsonl"
queue.write_text(json.dumps({"id": "only", "text": "once"}) + "\n")
stub = _StubSession()
speaker = RealtimeSpeaker(stub, queue_path=queue, processed_path=None)
def _stop():
return queue.read_text().strip() == ""
speaker.run_until_stopped(_stop, poll_interval=0.01)
assert stub.spoken == ["once"]
assert queue.read_text().strip() == ""
tests/plugins/test_kanban_attachments.py
+291
View File
@@ -0,0 +1,291 @@
"""Tests for Kanban task file attachments (#35338).
Covers three layers:
* ``hermes_cli.kanban_db`` accessors (add/list/get/delete + path helpers)
* the dashboard REST surface (upload / list / download / delete)
* worker-context surfacing so a kanban worker sees the absolute paths
The plugin router is attached to a bare FastAPI app — same approach as
``test_kanban_dashboard_plugin.py`` — so we exercise the real HTTP path
(multipart upload, streaming download) without the whole dashboard.
"""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from hermes_cli import kanban_db as kb
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _load_plugin_router():
repo_root = Path(__file__).resolve().parents[2]
plugin_file = repo_root / "plugins" / "kanban" / "dashboard" / "plugin_api.py"
assert plugin_file.exists(), f"plugin file missing: {plugin_file}"
spec = importlib.util.spec_from_file_location(
"hermes_dashboard_plugin_kanban_attach_test", plugin_file,
)
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = mod
spec.loader.exec_module(mod)
return mod.router
@pytest.fixture
def kanban_home(tmp_path, monkeypatch):
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
kb.init_db()
return home
@pytest.fixture
def client(kanban_home):
app = FastAPI()
app.include_router(_load_plugin_router(), prefix="/api/plugins/kanban")
return TestClient(app)
def _make_task(conn, title="t") -> str:
return kb.create_task(conn, title=title)
# ---------------------------------------------------------------------------
# DB-layer accessors
# ---------------------------------------------------------------------------
def test_add_list_get_delete_attachment(kanban_home, tmp_path):
conn = kb.connect()
try:
task_id = _make_task(conn)
# Write a real blob under the per-task dir so delete can unlink it.
dest_dir = kb.task_attachments_dir(task_id)
dest_dir.mkdir(parents=True, exist_ok=True)
blob = dest_dir / "source.pdf"
blob.write_bytes(b"%PDF-1.4 fake")
att_id = kb.add_attachment(
conn,
task_id,
filename="source.pdf",
stored_path=str(blob),
content_type="application/pdf",
size=blob.stat().st_size,
uploaded_by="tester",
)
assert att_id > 0
atts = kb.list_attachments(conn, task_id)
assert len(atts) == 1
a = atts[0]
assert a.filename == "source.pdf"
assert a.content_type == "application/pdf"
assert a.size == len(b"%PDF-1.4 fake")
assert a.uploaded_by == "tester"
assert a.stored_path == str(blob)
got = kb.get_attachment(conn, att_id)
assert got is not None and got.id == att_id
removed = kb.delete_attachment(conn, att_id)
assert removed is not None and removed.id == att_id
assert kb.list_attachments(conn, task_id) == []
assert not blob.exists(), "delete should unlink the on-disk blob"
assert kb.get_attachment(conn, att_id) is None
finally:
conn.close()
def test_add_attachment_rejects_unknown_task(kanban_home):
conn = kb.connect()
try:
with pytest.raises(ValueError):
kb.add_attachment(
conn, "t_doesnotexist", filename="x.txt", stored_path="/tmp/x.txt"
)
finally:
conn.close()
def test_add_attachment_appends_event(kanban_home):
conn = kb.connect()
try:
task_id = _make_task(conn)
kb.add_attachment(
conn, task_id, filename="a.txt", stored_path="/tmp/a.txt", size=3
)
kinds = [e.kind for e in kb.list_events(conn, task_id)]
assert "attached" in kinds
finally:
conn.close()
def test_delete_attachment_missing_returns_none(kanban_home):
conn = kb.connect()
try:
assert kb.delete_attachment(conn, 999999) is None
finally:
conn.close()
def test_attachments_root_is_per_board(kanban_home, monkeypatch):
# default board uses <root>/kanban/attachments
default_root = kb.attachments_root(board="default")
assert default_root.name == "attachments"
# a named board nests under its board dir
monkeypatch.delenv("HERMES_KANBAN_ATTACHMENTS_ROOT", raising=False)
named = kb.attachments_root(board="default")
assert named == default_root
def test_attachments_root_env_override(kanban_home, monkeypatch, tmp_path):
override = tmp_path / "custom-attach"
monkeypatch.setenv("HERMES_KANBAN_ATTACHMENTS_ROOT", str(override))
assert kb.attachments_root() == override
assert kb.task_attachments_dir("t_abc") == override / "t_abc"
# ---------------------------------------------------------------------------
# Worker context surfacing
# ---------------------------------------------------------------------------
def test_worker_context_lists_attachments_with_absolute_path(kanban_home):
conn = kb.connect()
try:
task_id = _make_task(conn, title="translate PDF")
dest_dir = kb.task_attachments_dir(task_id)
dest_dir.mkdir(parents=True, exist_ok=True)
blob = dest_dir / "manual.pdf"
blob.write_bytes(b"data")
kb.add_attachment(
conn,
task_id,
filename="manual.pdf",
stored_path=str(blob.resolve()),
content_type="application/pdf",
size=4,
)
ctx = kb.build_worker_context(conn, task_id)
assert "## Attachments" in ctx
assert "manual.pdf" in ctx
# The absolute path must appear so the worker can read_file it.
assert str(blob.resolve()) in ctx
finally:
conn.close()
def test_worker_context_no_attachments_section_when_empty(kanban_home):
conn = kb.connect()
try:
task_id = _make_task(conn)
ctx = kb.build_worker_context(conn, task_id)
assert "## Attachments" not in ctx
finally:
conn.close()
# ---------------------------------------------------------------------------
# REST surface — upload / list / download / delete round-trip
# ---------------------------------------------------------------------------
def _create_task_via_api(client) -> str:
r = client.post("/api/plugins/kanban/tasks", json={"title": "x"})
assert r.status_code == 200, r.text
return r.json()["task"]["id"]
def test_upload_list_download_delete_roundtrip(client):
task_id = _create_task_via_api(client)
content = b"hello attachment world"
# Upload
r = client.post(
f"/api/plugins/kanban/tasks/{task_id}/attachments",
files={"file": ("notes.txt", content, "text/plain")},
)
assert r.status_code == 200, r.text
att = r.json()["attachment"]
assert att["filename"] == "notes.txt"
assert att["size"] == len(content)
att_id = att["id"]
# List (drawer also embeds it in GET /tasks/:id)
r = client.get(f"/api/plugins/kanban/tasks/{task_id}/attachments")
assert r.status_code == 200
assert [a["filename"] for a in r.json()["attachments"]] == ["notes.txt"]
detail = client.get(f"/api/plugins/kanban/tasks/{task_id}").json()
assert "attachments" in detail
assert len(detail["attachments"]) == 1
# Download streams the exact bytes back
r = client.get(f"/api/plugins/kanban/attachments/{att_id}")
assert r.status_code == 200
assert r.content == content
# Delete removes the row and the file
r = client.delete(f"/api/plugins/kanban/attachments/{att_id}")
assert r.status_code == 200
assert client.get(f"/api/plugins/kanban/attachments/{att_id}").status_code == 404
assert client.get(
f"/api/plugins/kanban/tasks/{task_id}/attachments"
).json()["attachments"] == []
def test_upload_sanitizes_traversal_filename(client):
task_id = _create_task_via_api(client)
r = client.post(
f"/api/plugins/kanban/tasks/{task_id}/attachments",
files={"file": ("../../../../etc/passwd", b"x", "text/plain")},
)
assert r.status_code == 200, r.text
stored_path = r.json()["attachment"]["stored_path"]
# The leaf name only; never escapes the per-task attachments dir.
assert Path(stored_path).name == "passwd"
task_dir = kb.task_attachments_dir(task_id).resolve()
assert Path(stored_path).resolve().is_relative_to(task_dir)
def test_upload_name_collision_gets_suffixed(client):
task_id = _create_task_via_api(client)
for _ in range(2):
r = client.post(
f"/api/plugins/kanban/tasks/{task_id}/attachments",
files={"file": ("dup.txt", b"a", "text/plain")},
)
assert r.status_code == 200, r.text
names = sorted(
a["filename"]
for a in client.get(
f"/api/plugins/kanban/tasks/{task_id}/attachments"
).json()["attachments"]
)
assert names == ["dup (1).txt", "dup.txt"]
def test_upload_unknown_task_404(client):
r = client.post(
"/api/plugins/kanban/tasks/t_nope/attachments",
files={"file": ("x.txt", b"x", "text/plain")},
)
assert r.status_code == 404
def test_download_unknown_attachment_404(client):
assert client.get("/api/plugins/kanban/attachments/424242").status_code == 404
tests/plugins/test_kanban_dashboard_plugin.py
+2254
View File
File diff suppressed because it is too large Load Diff
tests/plugins/test_kanban_worker_runs.py
+440
View File
@@ -0,0 +1,440 @@
"""Tests for kanban worker/runs read endpoints.
Covers:
GET /workers/active
GET /runs/{run_id}
GET /runs/{run_id}/inspect
POST /runs/{run_id}/terminate
"""
from __future__ import annotations
import importlib.util
import secrets
import sys
import time
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from hermes_cli import kanban_db as kb
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _load_plugin_router():
"""Dynamically load plugins/kanban/dashboard/plugin_api.py and return its router."""
repo_root = Path(__file__).resolve().parents[2]
plugin_file = repo_root / "plugins" / "kanban" / "dashboard" / "plugin_api.py"
assert plugin_file.exists(), f"plugin file missing: {plugin_file}"
mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
# Re-use a cached module if already loaded to avoid duplicate-router issues.
if mod_name in sys.modules:
return sys.modules[mod_name].router
spec = importlib.util.spec_from_file_location(mod_name, plugin_file)
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
sys.modules[mod_name] = mod
spec.loader.exec_module(mod)
return mod.router
@pytest.fixture
def kanban_home(tmp_path, monkeypatch):
"""Isolated HERMES_HOME with an empty kanban DB."""
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
kb.init_db()
return home
@pytest.fixture
def client(kanban_home):
app = FastAPI()
app.include_router(_load_plugin_router(), prefix="/api/plugins/kanban")
return TestClient(app)
def _insert_run(conn, task_id, *, worker_pid=None, ended_at=None):
"""Insert a task_runs row directly (bypassing claim machinery) and return run_id."""
lock = secrets.token_hex(8)
future = int(time.time()) + 3600
cur = conn.execute(
"INSERT INTO task_runs "
"(task_id, status, claim_lock, claim_expires, worker_pid, started_at, ended_at) "
"VALUES (?, 'running', ?, ?, ?, ?, ?)",
(task_id, lock, future, worker_pid, int(time.time()), ended_at),
)
conn.commit()
return cur.lastrowid
# ---------------------------------------------------------------------------
# GET /workers/active
# ---------------------------------------------------------------------------
def test_workers_active_empty_board(client):
"""Board with no running tasks returns an empty workers list."""
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
body = r.json()
assert body["workers"] == []
assert body["count"] == 0
assert "checked_at" in body
def test_workers_active_with_running_task(client):
"""A running task with an open run row and worker_pid appears in the list."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="active-worker", assignee="alice")
conn.execute(
"UPDATE tasks SET status='running' WHERE id=?", (task_id,),
)
_insert_run(conn, task_id, worker_pid=12345)
finally:
conn.close()
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
body = r.json()
assert body["count"] == 1
w = body["workers"][0]
assert w["task_id"] == task_id
assert w["worker_pid"] == 12345
assert w["task_status"] == "running"
assert w["task_title"] == "active-worker"
assert w["task_assignee"] == "alice"
def test_workers_active_excludes_ended_runs(client):
"""Runs with ended_at set are excluded even if task is running."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ended-run", assignee="bob")
conn.execute("UPDATE tasks SET status='running' WHERE id=?", (task_id,))
_insert_run(conn, task_id, worker_pid=99999, ended_at=int(time.time()) - 60)
finally:
conn.close()
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
assert r.json()["count"] == 0
def test_workers_active_excludes_runs_without_pid(client):
"""Runs with no worker_pid are not considered active workers."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="no-pid", assignee="carol")
conn.execute("UPDATE tasks SET status='running' WHERE id=?", (task_id,))
_insert_run(conn, task_id, worker_pid=None)
finally:
conn.close()
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
assert r.json()["count"] == 0
# ---------------------------------------------------------------------------
# GET /runs/{run_id}
# ---------------------------------------------------------------------------
def test_get_run_404_unknown_id(client):
"""Non-existent run_id returns 404."""
r = client.get("/api/plugins/kanban/runs/999999")
assert r.status_code == 404
assert "999999" in r.json()["detail"]
def test_get_run_ok(client):
"""Existing run row returns 200 with expected shape."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="run-lookup", assignee="dave")
run_id = _insert_run(conn, task_id, worker_pid=55555)
finally:
conn.close()
r = client.get(f"/api/plugins/kanban/runs/{run_id}")
assert r.status_code == 200
body = r.json()
assert "run" in body
run = body["run"]
assert run["id"] == run_id
assert run["task_id"] == task_id
assert run["worker_pid"] == 55555
assert run["ended_at"] is None
# ---------------------------------------------------------------------------
# GET /runs/{run_id}/inspect
# ---------------------------------------------------------------------------
def test_inspect_run_404(client):
"""Non-existent run_id returns 404."""
r = client.get("/api/plugins/kanban/runs/888888/inspect")
assert r.status_code == 404
def test_inspect_run_already_ended(client):
"""Run with ended_at set returns alive=false with reason."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ended", assignee="eve")
run_id = _insert_run(conn, task_id, worker_pid=11111, ended_at=int(time.time()) - 10)
finally:
conn.close()
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is False
assert "ended" in body["reason"]
def test_inspect_run_no_pid(client):
"""Run with no worker_pid returns alive=false with reason."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="no-pid-inspect", assignee="frank")
run_id = _insert_run(conn, task_id, worker_pid=None)
finally:
conn.close()
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is False
assert "worker_pid" in body["reason"]
def test_inspect_run_dead_pid(client, monkeypatch):
"""Run with a non-existent PID returns alive=false via psutil.NoSuchProcess."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="dead-pid", assignee="grace")
run_id = _insert_run(conn, task_id, worker_pid=999999)
finally:
conn.close()
# Mock psutil to raise NoSuchProcess for any PID.
mock_psutil = MagicMock()
mock_psutil.NoSuchProcess = Exception
mock_psutil.AccessDenied = PermissionError
def _raise_no_such(*args, **kwargs):
raise mock_psutil.NoSuchProcess("no such process")
mock_psutil.Process = _raise_no_such
# Patch the module-level _psutil in the loaded plugin module.
plugin_mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
plugin_mod = sys.modules.get(plugin_mod_name)
if plugin_mod is not None:
monkeypatch.setattr(plugin_mod, "_psutil", mock_psutil)
else:
pytest.skip("plugin module not yet loaded")
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is False
assert body["pid"] == 999999
assert "not found" in body["reason"]
def test_inspect_run_live_pid(client, monkeypatch):
"""Run with a live PID returns alive=true with psutil fields."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="live-pid", assignee="heidi")
run_id = _insert_run(conn, task_id, worker_pid=12345)
finally:
conn.close()
# Build a realistic mock psutil.
mock_psutil = MagicMock()
mock_psutil.NoSuchProcess = type("NoSuchProcess", (Exception,), {})
mock_psutil.AccessDenied = type("AccessDenied", (Exception,), {})
fake_mem = MagicMock()
fake_mem.rss = 1024 * 1024 * 50 # 50 MB
fake_mem.vms = 1024 * 1024 * 200
fake_proc = MagicMock()
fake_proc.as_dict.return_value = {
"cpu_percent": 3.5,
"memory_info": fake_mem,
"num_threads": 4,
"status": "sleeping",
"create_time": time.time() - 300,
"cmdline": ["python", "-m", "hermes"],
}
fake_proc.num_fds.return_value = 12
mock_psutil.Process.return_value = fake_proc
plugin_mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
plugin_mod = sys.modules.get(plugin_mod_name)
if plugin_mod is not None:
monkeypatch.setattr(plugin_mod, "_psutil", mock_psutil)
else:
pytest.skip("plugin module not yet loaded")
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is True
assert body["pid"] == 12345
assert body["cpu_percent"] == 3.5
assert body["memory_rss_bytes"] == fake_mem.rss
assert body["num_threads"] == 4
assert body["status"] == "sleeping"
# ---------------------------------------------------------------------------
# POST /runs/{run_id}/terminate
# ---------------------------------------------------------------------------
def _setup_running_task_with_run(conn, *, title, assignee, worker_pid):
"""Create a task in 'running' state with a matching open task_runs row.
Mirrors what dispatcher_claim does: stamps tasks.status='running',
tasks.claim_lock, tasks.worker_pid; inserts task_runs row with the
same claim_lock so reclaim_task's preconditions are satisfied.
"""
task_id = kb.create_task(conn, title=title, assignee=assignee)
lock = secrets.token_hex(8)
future = int(time.time()) + 3600
conn.execute(
"UPDATE tasks SET status='running', claim_lock=?, "
"claim_expires=?, worker_pid=? WHERE id=?",
(lock, future, worker_pid, task_id),
)
cur = conn.execute(
"INSERT INTO task_runs "
"(task_id, status, claim_lock, claim_expires, worker_pid, started_at) "
"VALUES (?, 'running', ?, ?, ?, ?)",
(task_id, lock, future, worker_pid, int(time.time())),
)
conn.commit()
return task_id, cur.lastrowid
def test_terminate_run_404_unknown_id(client):
"""POST to unknown run_id returns 404."""
r = client.post(
"/api/plugins/kanban/runs/777777/terminate",
json={"reason": "test"},
)
assert r.status_code == 404
assert "777777" in r.json()["detail"]
def test_terminate_run_409_already_ended(client):
"""POST against a run with ended_at set returns 409."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ended-terminate", assignee="ivy")
run_id = _insert_run(
conn, task_id, worker_pid=22222, ended_at=int(time.time()) - 30,
)
finally:
conn.close()
r = client.post(
f"/api/plugins/kanban/runs/{run_id}/terminate",
json={"reason": "too late"},
)
assert r.status_code == 409
assert "already ended" in r.json()["detail"]
def test_terminate_run_ok(client, monkeypatch):
"""Happy path: live run is terminated, signal fn invoked, reason recorded."""
conn = kb.connect()
try:
task_id, run_id = _setup_running_task_with_run(
conn, title="kill-me", assignee="jane", worker_pid=33333,
)
finally:
conn.close()
# Capture signal calls so we don't actually SIGTERM a random PID.
sent = []
def _fake_terminate(pid, prev_lock, *, signal_fn=None):
sent.append((pid, prev_lock))
return {"signal": "SIGTERM", "delivered": True}
monkeypatch.setattr(kb, "_terminate_reclaimed_worker", _fake_terminate)
r = client.post(
f"/api/plugins/kanban/runs/{run_id}/terminate",
json={"reason": "operator abort"},
)
assert r.status_code == 200, r.text
body = r.json()
assert body == {"ok": True, "run_id": run_id, "task_id": task_id}
assert sent == [(33333, sent[0][1])]
assert sent[0][1] is not None # claim_lock was non-null
# Task is back to ready, claim cleared.
conn = kb.connect()
try:
row = conn.execute(
"SELECT status, claim_lock, worker_pid FROM tasks WHERE id=?",
(task_id,),
).fetchone()
finally:
conn.close()
assert row["status"] == "ready"
assert row["claim_lock"] is None
assert row["worker_pid"] is None
def test_terminate_run_409_task_not_reclaimable(client, monkeypatch):
"""Open run row whose task is no longer claimable returns 409."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ghost-run", assignee="ken")
# Task left in default 'ready' state with no claim_lock — task_run
# exists but reclaim_task will refuse because status != 'running'
# and claim_lock is NULL.
run_id = _insert_run(conn, task_id, worker_pid=44444)
finally:
conn.close()
# Make sure no signal is ever sent on this code path.
def _boom(*a, **k):
raise AssertionError("_terminate_reclaimed_worker should not be called")
monkeypatch.setattr(kb, "_terminate_reclaimed_worker", _boom)
r = client.post(
f"/api/plugins/kanban/runs/{run_id}/terminate",
json={"reason": "stale"},
)
assert r.status_code == 409
assert "reclaimable" in r.json()["detail"]
def test_terminate_run_accepts_empty_body(client):
"""Empty JSON body (no reason) is still accepted; falls through to 404."""
r = client.post(
"/api/plugins/kanban/runs/666666/terminate",
json={},
)
# 404 because run doesn't exist — what we're asserting here is that
# the endpoint doesn't 422 on a missing 'reason' field.
assert r.status_code == 404
tests/plugins/test_langfuse_plugin.py
+813
View File
@@ -0,0 +1,813 @@
"""Tests for the bundled observability/langfuse plugin."""
from __future__ import annotations
import importlib
import logging
import sys
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
PLUGIN_DIR = REPO_ROOT / "plugins" / "observability" / "langfuse"
# ---------------------------------------------------------------------------
# Manifest + layout
# ---------------------------------------------------------------------------
class TestManifest:
def test_plugin_directory_exists(self):
assert PLUGIN_DIR.is_dir()
assert (PLUGIN_DIR / "plugin.yaml").exists()
assert (PLUGIN_DIR / "__init__.py").exists()
def test_manifest_fields(self):
data = yaml.safe_load((PLUGIN_DIR / "plugin.yaml").read_text())
assert data["name"] == "langfuse"
assert data["version"]
# All six hooks the plugin implements.
assert set(data["hooks"]) == {
"pre_api_request", "post_api_request",
"pre_llm_call", "post_llm_call",
"pre_tool_call", "post_tool_call",
}
# Required env vars are the user-facing HERMES_ prefixed keys.
assert "HERMES_LANGFUSE_PUBLIC_KEY" in data["requires_env"]
assert "HERMES_LANGFUSE_SECRET_KEY" in data["requires_env"]
# ---------------------------------------------------------------------------
# Plugin discovery: langfuse is opt-in (not loaded unless explicitly enabled).
# This guards against someone accidentally re-introducing a per-hook
# load_config() gate or making the plugin auto-load.
# ---------------------------------------------------------------------------
class TestDiscovery:
def test_plugin_is_discovered_as_standalone_opt_in(self, tmp_path, monkeypatch):
"""Scanner should find the plugin but NOT load it by default."""
from hermes_cli import plugins as plugins_mod
# Isolated HERMES_HOME so we don't read the developer's config.yaml.
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
manager = plugins_mod.PluginManager()
manager.discover_and_load()
# observability/langfuse appears in the plugin registry …
loaded = manager._plugins.get("observability/langfuse")
assert loaded is not None, "plugin not discovered"
# … but is not loaded (opt-in default → no config.yaml means nothing enabled)
assert loaded.enabled is False
assert "not enabled" in (loaded.error or "").lower()
# ---------------------------------------------------------------------------
# Runtime gate: _get_langfuse() returns None and caches _INIT_FAILED when
# credentials are missing. Guards against regressing toward the rejected
# per-hook load_config() design.
# ---------------------------------------------------------------------------
class TestRuntimeGate:
def _fresh_plugin(self):
"""Import the plugin module fresh (clears any cached client)."""
mod_name = "plugins.observability.langfuse"
sys.modules.pop(mod_name, None)
return importlib.import_module(mod_name)
def test_get_langfuse_returns_none_without_credentials(self, monkeypatch):
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
langfuse_plugin = self._fresh_plugin()
assert langfuse_plugin._get_langfuse() is None
def test_get_langfuse_caches_failure_no_config_load(self, monkeypatch):
"""A miss must be cached — no per-hook config.yaml reads, no env re-reads."""
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
langfuse_plugin = self._fresh_plugin()
# Prime the cache with one call.
assert langfuse_plugin._get_langfuse() is None
# Now block os.environ.get — a correctly-cached plugin must not
# touch env again.
import os
called = {"n": 0}
real_get = os.environ.get
def tracking_get(key, default=None):
if key.startswith(("HERMES_LANGFUSE_", "LANGFUSE_")):
called["n"] += 1
return real_get(key, default)
monkeypatch.setattr(os.environ, "get", tracking_get)
for _ in range(20):
assert langfuse_plugin._get_langfuse() is None
assert called["n"] == 0, (
f"_get_langfuse() re-read env {called['n']} times after cache miss — "
"it should short-circuit via _INIT_FAILED"
)
def test_get_langfuse_does_not_import_hermes_config(self, monkeypatch):
"""The plugin must not re-read config.yaml per hook."""
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
# Drop any cached import of hermes_cli.config.
sys.modules.pop("hermes_cli.config", None)
langfuse_plugin = self._fresh_plugin()
for _ in range(20):
langfuse_plugin._get_langfuse()
assert "hermes_cli.config" not in sys.modules, (
"langfuse plugin imported hermes_cli.config — regression toward "
"the rejected per-hook load_config() design"
)
# ---------------------------------------------------------------------------
# Hooks are inert when the client is unavailable.
# ---------------------------------------------------------------------------
class TestHooksInert:
def test_hooks_noop_without_client(self, monkeypatch):
"""All 6 hooks must return without raising when _get_langfuse() is None."""
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
sys.modules.pop("plugins.observability.langfuse", None)
import importlib
mod = importlib.import_module("plugins.observability.langfuse")
# Each hook should just return; no exceptions.
mod.on_pre_llm_call(task_id="t", session_id="s", messages=[{"role": "user", "content": "hi"}])
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, request_messages=[])
mod.on_post_llm_call(task_id="t", session_id="s", api_call_count=1)
mod.on_pre_tool_call(tool_name="read_file", args={}, task_id="t", session_id="s")
mod.on_post_tool_call(tool_name="read_file", args={}, result="ok", task_id="t", session_id="s")
class TestPayloadSanitization:
def test_safe_value_redacts_base64_data_uri_instead_of_truncating(self):
sys.modules.pop("plugins.observability.langfuse", None)
import importlib
mod = importlib.import_module("plugins.observability.langfuse")
payload = "data:image/png;base64," + ("a" * 20000)
result = mod._safe_value(payload)
assert result == {
"type": "data_uri",
"media_type": "image/png",
"omitted": True,
"length": len(payload),
}
def test_serialize_messages_redacts_data_uri_parts(self):
sys.modules.pop("plugins.observability.langfuse", None)
import importlib
mod = importlib.import_module("plugins.observability.langfuse")
payload = "data:image/jpeg;base64," + ("b" * 20000)
serialized = mod._serialize_messages([
{"role": "user", "content": [{"type": "image_url", "image_url": {"url": payload}}]}
])
assert serialized[0]["content"][0]["image_url"]["url"] == {
"type": "data_uri",
"media_type": "image/jpeg",
"omitted": True,
"length": len(payload),
}
# ---------------------------------------------------------------------------
# Placeholder-credential guard (#23823).
#
# Regression coverage for the silent-failure bug: when an operator leaves
# HERMES_LANGFUSE_PUBLIC_KEY / SECRET_KEY at a template value like
# "placeholder", "test-key", or "your-langfuse-key", the SDK accepts the
# credentials at construction time (it does no server-side validation
# eagerly) but drops every trace at flush time, with no signal in the
# Hermes logs. The fix in `_get_langfuse()` validates the documented
# `pk-lf-` / `sk-lf-` prefix Langfuse always issues, surfaces a one-shot
# warning naming the offending env var(s), and short-circuits via the
# same `_INIT_FAILED` path used for missing credentials so subsequent
# hook invocations don't re-log.
# ---------------------------------------------------------------------------
class _FakeLangfuse:
"""Stand-in for the real :class:`langfuse.Langfuse` so tests don't
need the optional ``langfuse`` SDK installed. The plugin's runtime
gate refuses to proceed past ``if Langfuse is None`` when the SDK
is missing, which would short-circuit before the placeholder check
can fire. Patching ``plugin.Langfuse`` with this class lets the
placeholder validator exercise its full code path."""
instances: list["_FakeLangfuse"] = []
def __init__(self, **kwargs):
self.kwargs = kwargs
_FakeLangfuse.instances.append(self)
class TestPlaceholderKeyDetection:
LOGGER_NAME = "plugins.observability.langfuse"
def _fresh_plugin(self, monkeypatch=None):
mod_name = "plugins.observability.langfuse"
sys.modules.pop(mod_name, None)
mod = importlib.import_module(mod_name)
if monkeypatch is not None:
# Pretend the SDK is installed so `_get_langfuse()` actually
# reaches the placeholder check. Real SDK calls are never
# made because the placeholder/missing-credentials paths
# return before constructing a client.
_FakeLangfuse.instances.clear()
monkeypatch.setattr(mod, "Langfuse", _FakeLangfuse, raising=False)
return mod
@staticmethod
def _clear_env(monkeypatch):
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
# -- helper unit tests (no SDK stub needed: these don't go through
# _get_langfuse, they exercise the pure-Python helpers directly) ------
def test_redact_key_preview_empty(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._redact_key_preview("") == "<empty>"
def test_redact_key_preview_short_value_echoed(self, monkeypatch):
"""Short placeholder strings are echoed in full so the operator
can see exactly which template they forgot to replace."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._redact_key_preview("placeholder") == "'placeholder'"
assert plugin._redact_key_preview("test-key") == "'test-key'"
def test_redact_key_preview_long_value_truncated(self, monkeypatch):
"""If an operator pasted a real secret into the wrong env var the
preview must NOT echo it in full — only the leading 6 chars."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
result = plugin._redact_key_preview("sk-lf-abcdefghijklmnop")
assert "abcdefghij" not in result
assert result.startswith("'sk-lf-")
assert result.endswith("...'")
def test_validate_langfuse_key_accepts_documented_prefix(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._validate_langfuse_key(
"HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz"
) is None
assert plugin._validate_langfuse_key(
"HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz"
) is None
def test_validate_langfuse_key_rejects_wrong_prefix(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
msg = plugin._validate_langfuse_key(
"HERMES_LANGFUSE_PUBLIC_KEY", "placeholder"
)
assert msg is not None
assert "HERMES_LANGFUSE_PUBLIC_KEY" in msg
assert "pk-lf-" in msg
def test_validate_langfuse_key_unknown_name_passes(self, monkeypatch):
"""Defensive: an env var with no registered prefix is trusted."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._validate_langfuse_key("HERMES_LANGFUSE_BASE_URL", "anything") is None
# -- end-to-end _get_langfuse() behaviour --------------------------------
# These tests pass `monkeypatch` to _fresh_plugin() so the helper can
# stub out `Langfuse` (the optional SDK). Without that, every call
# short-circuits at `if Langfuse is None` before reaching the
# placeholder validator — masking the very behaviour we're testing.
def test_placeholder_public_key_warns_and_skips(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
text = caplog.text
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
assert "'placeholder'" in text
assert "pk-lf-" in text
# The valid secret value must NOT appear (the var NAME does, in
# the "or unset ..." hint, but the value preview shouldn't).
assert "'sk-lf-" not in text
# Never constructed the SDK client — short-circuited before that.
assert _FakeLangfuse.instances == []
def test_placeholder_secret_key_warns_and_skips(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "test-key")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
text = caplog.text
assert "HERMES_LANGFUSE_SECRET_KEY" in text
assert "'test-key'" in text
assert "sk-lf-" in text
# The valid public value must NOT appear.
assert "'pk-lf-" not in text
assert _FakeLangfuse.instances == []
def test_both_placeholders_one_warning_with_both_keys(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert len(warnings) == 1, (
f"Expected a single combined warning; got {len(warnings)}:\n"
+ "\n".join(r.getMessage() for r in warnings)
)
text = warnings[0].getMessage()
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
assert "HERMES_LANGFUSE_SECRET_KEY" in text
def test_repeated_calls_do_not_re_warn(self, monkeypatch, caplog):
"""The cached ``_INIT_FAILED`` sentinel must short-circuit
subsequent calls so each hook invocation isn't a fresh log
line — otherwise a busy gateway will spam the operator's
terminal."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
for _ in range(15):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert len(warnings) == 1, (
f"Warning fired {len(warnings)} times across 15 calls; "
"expected 1 (cached via _INIT_FAILED)"
)
@pytest.mark.parametrize("placeholder", [
"placeholder",
"test-key",
"your-langfuse-key",
"change-me",
"xxx",
"dummy-key-here",
"<your-key>",
"REPLACE_ME",
])
def test_common_placeholders_detected(self, monkeypatch, caplog, placeholder):
"""A grab-bag of values that real-world ``.env.example`` templates
use as stand-ins. Any of them in either key must trip the guard."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", placeholder)
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
def test_legacy_LANGFUSE_PUBLIC_KEY_also_validated(self, monkeypatch, caplog):
"""The plugin reads both the canonical HERMES_-prefixed env var and
the legacy bare ``LANGFUSE_PUBLIC_KEY``. The validator must run on
whichever value ``_get_langfuse()`` actually consumed."""
self._clear_env(monkeypatch)
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
# Warning names the canonical user-facing env var (the bare
# LANGFUSE_PUBLIC_KEY is a backwards-compat alias for the
# HERMES_-prefixed one — operators set the HERMES_-prefixed one).
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
assert "'placeholder'" in caplog.text
def test_missing_credentials_still_skip_silently(self, monkeypatch, caplog):
"""Missing-creds is the documented opt-out path (operator hasn't
configured the plugin yet) — it must remain SILENT. Regression
guard against the placeholder validator accidentally running on
empty values and re-introducing log noise for unconfigured
installs."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert warnings == []
def test_sdk_not_installed_still_skips_silently(self, monkeypatch, caplog):
"""If the langfuse SDK isn't installed at all, the placeholder
check should never run — there's nothing the operator can do
about a credential mismatch when the package is missing, and
re-warning here would dilute the actually-actionable SDK-missing
signal upstream. The ``Langfuse is None`` guard at the top of
``_get_langfuse`` already handles this; this test pins that
behaviour."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
# NO monkeypatch on Langfuse here — falls back to whatever the
# plugin imported at module load (None if SDK absent).
plugin = self._fresh_plugin()
monkeypatch.setattr(plugin, "Langfuse", None, raising=False)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert warnings == []
def test_valid_prefixes_do_not_trigger_placeholder_warning(self, monkeypatch, caplog):
"""Real Langfuse keys (``pk-lf-…`` / ``sk-lf-…``) must pass the
guard and proceed to SDK init. We stub the SDK constructor with
a recording fake so the assertion can confirm BOTH that the
placeholder warning didn't fire AND that the client was actually
constructed — the latter is the success signal the bug report
wanted."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
client = plugin._get_langfuse()
assert isinstance(client, _FakeLangfuse)
assert client.kwargs["public_key"] == "pk-lf-real-public-xyz"
assert client.kwargs["secret_key"] == "sk-lf-real-secret-xyz"
assert "placeholders" not in caplog.text.lower(), (
f"Valid Langfuse keys tripped the placeholder guard: {caplog.text!r}"
)
class TestRequestMessageCoercion:
def test_prefers_request_messages_then_messages_then_history_then_user_message(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
assert mod._coerce_request_messages(
request_messages=[{"role": "system", "content": "s"}],
messages=[{"role": "user", "content": "m"}],
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "system", "content": "s"}]
assert mod._coerce_request_messages(
messages=[{"role": "user", "content": "m"}],
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "user", "content": "m"}]
assert mod._coerce_request_messages(
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "user", "content": "h"}]
assert mod._coerce_request_messages(user_message="u") == [{"role": "user", "content": "u"}]
class TestToolCallOutputBackfill:
def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
observation = object()
state = mod.TraceState(trace_id="trace-1", root_ctx=None, root_span=None)
state.tools["call-1"] = observation
state.turn_tool_calls.append({
"id": "call-1",
"type": "function",
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
"function": {
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
},
})
task_key = mod._trace_key("task-1", "session-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):
ended["observation"] = obs
ended["output"] = output
ended["metadata"] = metadata
monkeypatch.setattr(mod, "_end_observation", fake_end_observation)
mod.on_post_tool_call(
tool_name="web_extract",
args={"urls": ["https://example.com"]},
result='{"results": [{"url": "https://example.com", "content": "Example Domain"}]}',
task_id="task-1",
session_id="session-1",
tool_call_id="call-1",
)
assert ended["observation"] is observation
assert state.turn_tool_calls[0]["output"] == ended["output"]
assert state.turn_tool_calls[0]["function"]["output"] == ended["output"]
assert state.turn_tool_calls[0]["output"] == {
"results": [{"url": "https://example.com", "content": "Example Domain"}]
}
def test_serialize_messages_keeps_tool_name_and_call_id(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
messages = [{
"role": "tool",
"name": "web_extract",
"tool_call_id": "call-1",
"content": '{"ok": true}',
}]
assert mod._serialize_messages(messages) == [{
"role": "tool",
"name": "web_extract",
"tool_call_id": "call-1",
"content": {"ok": True},
}]
def test_serialize_tool_calls_emits_openai_style_function_shape(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
class _Fn:
name = "web_extract"
arguments = '{"urls": ["https://example.com"]}'
class _ToolCall:
id = "call-1"
type = "function"
function = _Fn()
assert mod._serialize_tool_calls([_ToolCall()]) == [{
"id": "call-1",
"type": "function",
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
"function": {
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
},
}]
class TestToolObservationKeying:
"""Tests for pre/post tool_call observation matching when tool_call_id is absent."""
def _make_mod(self):
sys.modules.pop("plugins.observability.langfuse", None)
return importlib.import_module("plugins.observability.langfuse")
def test_empty_tool_call_id_single_tool_sets_output(self, monkeypatch):
mod = self._make_mod()
obs = object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name.setdefault("my_tool", []).append(obs)
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end(o, *, output=None, metadata=None, **kw):
ended["obs"] = o
ended["output"] = output
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="my_tool",
args={},
result='{"ok": true}',
task_id="task-1",
session_id="sess-1",
tool_call_id="",
)
assert ended["obs"] is obs
assert ended["output"] == {"ok": True}
assert state.pending_tools_by_name.get("my_tool") is None
def test_empty_tool_call_id_observations_are_fifo_within_tool_name(self, monkeypatch):
"""Two queued observations are consumed in FIFO order so the first
post hook gets the first observation's output, not the second.
Sequential-on-one-thread coverage; the real concurrent case is
guarded by ``_STATE_LOCK`` around every read-modify-write on
``pending_tools_by_name`` and is exercised in
``test_threaded_post_calls_preserve_fifo_under_lock`` below.
"""
mod = self._make_mod()
obs_a, obs_b = object(), object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name["web_extract"] = [obs_a, obs_b]
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
calls = []
def fake_end(o, *, output=None, metadata=None, **kw):
calls.append((o, output))
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"val": "a"}',
task_id="task-1", session_id="sess-1", tool_call_id="",
)
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"val": "b"}',
task_id="task-1", session_id="sess-1", tool_call_id="",
)
assert calls[0] == (obs_a, {"val": "a"})
assert calls[1] == (obs_b, {"val": "b"})
assert state.pending_tools_by_name.get("web_extract") is None
def test_threaded_post_calls_preserve_fifo_under_lock(self, monkeypatch):
"""The actual concurrency contract: when 8 threads race to drain
the pending queue, no observation is consumed twice and none is
lost. Validates ``_STATE_LOCK`` discipline, not Python list
semantics."""
import threading
mod = self._make_mod()
n = 8
observations = [object() for _ in range(n)]
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name["web_extract"] = list(observations)
task_key = mod._trace_key("task-thr", "sess-thr")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
recorded: list = []
lock = threading.Lock()
def fake_end(o, *, output=None, metadata=None, **kw):
with lock:
recorded.append(o)
monkeypatch.setattr(mod, "_end_observation", fake_end)
barrier = threading.Barrier(n)
def worker():
barrier.wait()
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"ok": true}',
task_id="task-thr", session_id="sess-thr", tool_call_id="",
)
threads = [threading.Thread(target=worker) for _ in range(n)]
for t in threads:
t.start()
for t in threads:
t.join()
# Every observation was consumed exactly once; queue is empty.
assert len(recorded) == n
assert set(map(id, recorded)) == set(map(id, observations))
assert state.pending_tools_by_name.get("web_extract") is None
def test_explicit_tool_call_id_uses_tools_dict(self, monkeypatch):
"""When tool_call_id is present, pending_tools_by_name is not touched."""
mod = self._make_mod()
obs = object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.tools["call-99"] = obs
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end(o, *, output=None, metadata=None, **kw):
ended["obs"] = o
ended["output"] = output
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="my_tool", args={}, result='{"status": "done"}',
task_id="task-1", session_id="sess-1", tool_call_id="call-99",
)
assert ended["obs"] is obs
assert ended["output"] == {"status": "done"}
assert not state.tools
class TestUsageFromSanitizedResponse:
"""Regression: ``post_api_request`` delivers ``response`` as a sanitized
dict (no ``.usage`` attribute) plus a separate ``usage`` summary dict. The
post-call handler must read the ``usage`` dict instead of treating the dict
response as a usage-bearing object and dropping all token/cost data."""
def _setup(self, mod, monkeypatch):
# Active client so on_post_llm_call does not early-return.
monkeypatch.setattr(mod, "_get_langfuse", lambda: object())
observation = object()
state = mod.TraceState(trace_id="trace-1", root_ctx=None, root_span=None)
state.generations[mod._request_key(1)] = observation
monkeypatch.setitem(mod._TRACE_STATE, mod._trace_key("task-1", "session-1"), state)
captured = {}
def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):
captured["usage_details"] = usage_details
monkeypatch.setattr(mod, "_end_observation", fake_end_observation)
return captured
def test_sanitized_dict_response_uses_usage_dict(self, monkeypatch):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
captured = self._setup(mod, monkeypatch)
# A plain dict has no ``.usage`` attribute — mirrors post_api_request.
mod.on_post_llm_call(
task_id="task-1",
session_id="session-1",
api_call_count=1,
model="gemini-3-flash-preview",
response={"model": "gemini-3-flash-preview", "usage": {"input_tokens": 100, "output_tokens": 20}},
usage={"input_tokens": 100, "output_tokens": 20},
assistant_content_chars=42,
)
# Before the fix the dict response shadowed the usage dict and tokens
# were lost (usage_details == {}).
assert captured["usage_details"] == {"input": 100, "output": 20}
def test_real_response_object_with_usage_still_used(self, monkeypatch):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
captured = self._setup(mod, monkeypatch)
# A response object that genuinely carries usage must still take the
# response-object path (post_llm_call / legacy behavior).
seen = {}
def fake_usage_and_cost(resp, **_):
seen["resp"] = resp
return {"input": 7, "output": 3}, {}
monkeypatch.setattr(mod, "_usage_and_cost", fake_usage_and_cost)
class _Resp:
usage = {"prompt_tokens": 7, "completion_tokens": 3}
resp = _Resp()
mod.on_post_llm_call(
task_id="task-1",
session_id="session-1",
api_call_count=1,
model="gemini-3-flash-preview",
response=resp,
usage={"input_tokens": 999, "output_tokens": 999},
assistant_content_chars=42,
)
assert seen["resp"] is resp
assert captured["usage_details"] == {"input": 7, "output": 3}
tests/plugins/test_nemo_relay_plugin.py
+1376
View File
File diff suppressed because it is too large Load Diff
tests/plugins/test_plugin_dashboard_auth_contract.py
+95
View File
@@ -0,0 +1,95 @@
"""Guardrail: dashboard plugins must NOT read the session token directly.
The dashboard host exposes a sanctioned, gated-mode-aware auth surface on the
plugin SDK (``window.__HERMES_PLUGIN_SDK__``): ``fetchJSON`` (JSON REST),
``authedFetch`` (uploads / blob downloads), and ``buildWsUrl`` /
``buildWsAuthParam`` (WebSockets). These handle BOTH dashboard auth modes —
loopback (``X-Hermes-Session-Token`` header) and gated OAuth
(``hermes_session_at`` cookie / single-use ``?ticket=``).
Plugins that hand-roll ``fetch`` / ``WebSocket`` and read
``window.__HERMES_SESSION_TOKEN__`` directly send an empty token in gated mode
and 401/1008. That bug shipped in the kanban and achievements plugins and was
invisible until the dashboard ran gated on hosted Fly agents.
This test fails if any bundled plugin's frontend reads the token global
directly, forcing new/edited plugins through the SDK surface instead. It is
the enforcement half of the "single sanctioned auth surface" design — the SDK
helpers are the carrot, this test is the stick.
If you have a legitimate reason to reference the token name (e.g. a comment
explaining why NOT to use it), add the file to ``_ALLOWED_FILES`` with a note.
"""
from __future__ import annotations
import re
from pathlib import Path
import pytest
# Repo root: tests/plugins/<this file> → ../../
_REPO_ROOT = Path(__file__).resolve().parents[2]
_PLUGINS_DIR = _REPO_ROOT / "plugins"
# The forbidden global. Reading it directly bypasses the gated-mode auth path.
_FORBIDDEN = "__HERMES_SESSION_TOKEN__"
# Files explicitly allowed to mention the token (none today). Map path →
# reason so the allowance is self-documenting if one is ever needed.
_ALLOWED_FILES: dict[str, str] = {}
def _plugin_frontend_bundles() -> list[Path]:
"""Every plugin-shipped JS bundle the dashboard loads into the browser."""
if not _PLUGINS_DIR.is_dir():
return []
# Plugin dashboards live at plugins/<name>/dashboard/dist/*.js
return sorted(_PLUGINS_DIR.glob("*/dashboard/dist/*.js"))
def test_there_are_plugin_bundles_to_check() -> None:
"""Sanity: the glob actually finds the bundles, so a future layout change
doesn't silently turn this guard into a no-op."""
bundles = _plugin_frontend_bundles()
names = {b.parent.parent.parent.name for b in bundles}
# kanban + hermes-achievements are bundled today; assert at least one is
# found so the guard can't pass vacuously.
assert bundles, "no plugin dashboard bundles found — glob/layout drift?"
assert names, "could not resolve plugin names from bundle paths"
@pytest.mark.parametrize(
"bundle",
_plugin_frontend_bundles(),
ids=lambda p: str(p.relative_to(_REPO_ROOT)),
)
def test_plugin_bundle_does_not_read_session_token(bundle: Path) -> None:
rel = str(bundle.relative_to(_REPO_ROOT))
text = bundle.read_text(encoding="utf-8", errors="replace")
if rel in _ALLOWED_FILES:
return # explicitly allowed (with a documented reason)
# Only flag CODE reads of the token global, not mentions in ``//`` comments
# (e.g. a comment explaining why the SDK helper is used instead). A line is
# a code read if it contains the global and the global appears before any
# ``//`` comment marker on that line.
offending: list[str] = []
for i, line in enumerate(text.splitlines(), start=1):
idx = line.find(_FORBIDDEN)
if idx == -1:
continue
comment_idx = line.find("//")
in_comment = comment_idx != -1 and comment_idx < idx
if not in_comment:
offending.append(f" {i}: {line.strip()}")
if not offending:
return
pytest.fail(
f"{rel} reads {_FORBIDDEN} directly — this bypasses gated-mode auth "
f"and 401/1008s on OAuth-gated dashboards. Use the plugin SDK instead: "
f"SDK.fetchJSON (JSON), SDK.authedFetch (uploads/downloads), or "
f"SDK.buildWsUrl (WebSockets). Offending lines:\n" + "\n".join(offending)
)
tests/plugins/test_retaindb_plugin.py
+738
View File
@@ -0,0 +1,738 @@
"""Tests for the RetainDB memory plugin.
Covers: _Client HTTP client, _WriteQueue SQLite queue, _build_overlay formatter,
RetainDBMemoryProvider lifecycle/tools/prefetch, thread management, connection pooling.
"""
import json
import sqlite3
import time
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Imports — guarded since plugins/memory lives outside the standard test path
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(tmp_path, monkeypatch):
"""Ensure HERMES_HOME and RETAINDB vars are isolated."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.delenv("RETAINDB_API_KEY", raising=False)
monkeypatch.delenv("RETAINDB_BASE_URL", raising=False)
monkeypatch.delenv("RETAINDB_PROJECT", raising=False)
@pytest.fixture(autouse=True)
def _cap_retaindb_sleeps(monkeypatch):
"""Cap production-code sleeps so background-thread tests run fast.
The retaindb ``_WriteQueue._flush_row`` does ``time.sleep(2)`` after
errors. Across multiple tests that trigger the retry path, that adds
up. Cap the module's bound ``time.sleep`` to 0.05s — tests don't care
about the exact retry delay, only that it happens. The test file's
own ``time.sleep`` stays real since it uses a different reference.
"""
try:
from plugins.memory import retaindb as _retaindb
except ImportError:
return
real_sleep = _retaindb.time.sleep
def _capped_sleep(seconds):
return real_sleep(min(float(seconds), 0.05))
import types as _types
fake_time = _types.SimpleNamespace(sleep=_capped_sleep, time=_retaindb.time.time)
monkeypatch.setattr(_retaindb, "time", fake_time)
# We need the repo root on sys.path so the plugin can import agent.memory_provider
import sys
_repo_root = str(Path(__file__).resolve().parents[2])
if _repo_root not in sys.path:
sys.path.insert(0, _repo_root)
from plugins.memory.retaindb import (
_Client,
_WriteQueue,
_build_overlay,
RetainDBMemoryProvider,
)
# ===========================================================================
# _Client tests
# ===========================================================================
class TestClient:
"""Test the HTTP client with mocked requests."""
def _make_client(self, api_key="rdb-test-key", base_url="https://api.retaindb.com", project="test"):
return _Client(api_key, base_url, project)
def test_base_url_trailing_slash_stripped(self):
c = self._make_client(base_url="https://api.retaindb.com///")
assert c.base_url == "https://api.retaindb.com"
def test_headers_include_auth(self):
c = self._make_client()
h = c._headers("/v1/files")
assert h["Authorization"] == "Bearer rdb-test-key"
assert "X-API-Key" not in h
def test_headers_include_api_key_for_memory_path(self):
c = self._make_client()
h = c._headers("/v1/memory/search")
assert h["X-API-Key"] == "rdb-test-key"
def test_headers_include_api_key_for_context_path(self):
c = self._make_client()
h = c._headers("/v1/context/query")
assert h["X-API-Key"] == "rdb-test-key"
def test_headers_strip_bearer_prefix(self):
c = self._make_client(api_key="Bearer rdb-test-key")
h = c._headers("/v1/memory/search")
assert h["Authorization"] == "Bearer rdb-test-key"
assert h["X-API-Key"] == "rdb-test-key"
def test_add_memory_tries_fallback(self):
c = self._make_client()
call_count = 0
def fake_request(method, path, **kwargs):
nonlocal call_count
call_count += 1
if call_count == 1:
raise RuntimeError("404")
return {"id": "mem-1"}
with patch.object(c, "request", side_effect=fake_request):
result = c.add_memory("u1", "s1", "test fact")
assert result == {"id": "mem-1"}
assert call_count == 2
def test_delete_memory_tries_fallback(self):
c = self._make_client()
call_count = 0
def fake_request(method, path, **kwargs):
nonlocal call_count
call_count += 1
if call_count == 1:
raise RuntimeError("404")
return {"deleted": True}
with patch.object(c, "request", side_effect=fake_request):
result = c.delete_memory("mem-123")
assert result == {"deleted": True}
assert call_count == 2
# ===========================================================================
# _WriteQueue tests
# ===========================================================================
class TestWriteQueue:
"""Test the SQLite-backed write queue with real SQLite."""
def _make_queue(self, tmp_path, client=None):
if client is None:
client = MagicMock()
client.ingest_session = MagicMock(return_value={"status": "ok"})
db_path = tmp_path / "test_queue.db"
return _WriteQueue(client, db_path), client, db_path
def test_enqueue_creates_row(self, tmp_path):
q, client, db_path = self._make_queue(tmp_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
# shutdown() blocks until the writer thread drains the queue — no need
# to pre-sleep (the old 1s sleep was a just-in-case wait, but shutdown
# does the right thing).
q.shutdown()
# If ingest succeeded, the row should be deleted
client.ingest_session.assert_called_once()
def test_enqueue_persists_to_sqlite(self, tmp_path):
client = MagicMock()
# Make ingest slow so the row is still in SQLite when we peek.
# 0.5s is plenty — the test just needs the flush to still be in-flight.
client.ingest_session = MagicMock(side_effect=lambda *a, **kw: time.sleep(0.5))
db_path = tmp_path / "test_queue.db"
q = _WriteQueue(client, db_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "test"}])
# Check SQLite directly — row should exist since flush is slow
conn = sqlite3.connect(str(db_path))
rows = conn.execute("SELECT user_id, session_id FROM pending").fetchall()
conn.close()
assert len(rows) >= 1
assert rows[0][0] == "user1"
q.shutdown()
def test_flush_deletes_row_on_success(self, tmp_path):
q, client, db_path = self._make_queue(tmp_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
q.shutdown() # blocks until drain
# Row should be gone
conn = sqlite3.connect(str(db_path))
rows = conn.execute("SELECT COUNT(*) FROM pending").fetchone()[0]
conn.close()
assert rows == 0
def test_flush_records_error_on_failure(self, tmp_path):
client = MagicMock()
client.ingest_session = MagicMock(side_effect=RuntimeError("API down"))
db_path = tmp_path / "test_queue.db"
q = _WriteQueue(client, db_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
# Poll for the error to be recorded (max 2s), instead of a fixed 3s wait.
deadline = time.time() + 2.0
last_error = None
while time.time() < deadline:
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT last_error FROM pending").fetchone()
conn.close()
if row and row[0]:
last_error = row[0]
break
time.sleep(0.05)
q.shutdown()
assert last_error is not None
assert "API down" in last_error
def test_thread_local_connection_reuse(self, tmp_path):
q, _, _ = self._make_queue(tmp_path)
# Same thread should get same connection
conn1 = q._get_conn()
conn2 = q._get_conn()
assert conn1 is conn2
q.shutdown()
def test_crash_recovery_replays_pending(self, tmp_path):
"""Simulate crash: create rows, then new queue should replay them."""
db_path = tmp_path / "recovery_test.db"
# First: create a queue and insert rows, but don't let them flush
client1 = MagicMock()
client1.ingest_session = MagicMock(side_effect=RuntimeError("fail"))
q1 = _WriteQueue(client1, db_path)
q1.enqueue("user1", "sess1", [{"role": "user", "content": "lost turn"}])
# Wait until the error is recorded (poll with short interval).
deadline = time.time() + 2.0
while time.time() < deadline:
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT last_error FROM pending").fetchone()
conn.close()
if row and row[0]:
break
time.sleep(0.05)
q1.shutdown()
# Now create a new queue — it should replay the pending rows
client2 = MagicMock()
client2.ingest_session = MagicMock(return_value={"status": "ok"})
q2 = _WriteQueue(client2, db_path)
# Poll for the replay to happen.
deadline = time.time() + 2.0
while time.time() < deadline:
if client2.ingest_session.called:
break
time.sleep(0.05)
q2.shutdown()
# The replayed row should have been ingested via client2
client2.ingest_session.assert_called_once()
call_args = client2.ingest_session.call_args
assert call_args[0][0] == "user1" # user_id
# ===========================================================================
# _build_overlay tests
# ===========================================================================
class TestBuildOverlay:
"""Test the overlay formatter (pure function)."""
def test_empty_inputs_returns_empty(self):
assert _build_overlay({}, {}) == ""
def test_empty_memories_returns_empty(self):
assert _build_overlay({"memories": []}, {"results": []}) == ""
def test_profile_items_included(self):
profile = {"memories": [{"content": "User likes Python"}]}
result = _build_overlay(profile, {})
assert "User likes Python" in result
assert "[RetainDB Context]" in result
def test_query_results_included(self):
query_result = {"results": [{"content": "Previous discussion about Rust"}]}
result = _build_overlay({}, query_result)
assert "Previous discussion about Rust" in result
def test_deduplication_removes_duplicates(self):
profile = {"memories": [{"content": "User likes Python"}]}
query_result = {"results": [{"content": "User likes Python"}]}
result = _build_overlay(profile, query_result)
assert result.count("User likes Python") == 1
def test_local_entries_filter(self):
profile = {"memories": [{"content": "Already known fact"}]}
result = _build_overlay(profile, {}, local_entries=["Already known fact"])
# The profile item matches a local entry, should be filtered
assert result == ""
def test_max_five_items_per_section(self):
profile = {"memories": [{"content": f"Fact {i}"} for i in range(10)]}
result = _build_overlay(profile, {})
# Should only include first 5
assert "Fact 0" in result
assert "Fact 4" in result
assert "Fact 5" not in result
def test_none_content_handled(self):
profile = {"memories": [{"content": None}, {"content": "Real fact"}]}
result = _build_overlay(profile, {})
assert "Real fact" in result
def test_truncation_at_320_chars(self):
long_content = "x" * 500
profile = {"memories": [{"content": long_content}]}
result = _build_overlay(profile, {})
# Each item is compacted to 320 chars max
for line in result.split("\n"):
if line.startswith("- "):
assert len(line) <= 322 # "- " + 320
# ===========================================================================
# RetainDBMemoryProvider tests
# ===========================================================================
class TestRetainDBMemoryProvider:
"""Test the main plugin class."""
def _make_provider(self, tmp_path, monkeypatch, api_key="rdb-test-key"):
monkeypatch.setenv("RETAINDB_API_KEY", api_key)
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
(tmp_path / ".hermes").mkdir(exist_ok=True)
provider = RetainDBMemoryProvider()
return provider
def test_name(self):
p = RetainDBMemoryProvider()
assert p.name == "retaindb"
def test_is_available_without_key(self):
p = RetainDBMemoryProvider()
assert p.is_available() is False
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test")
p = RetainDBMemoryProvider()
assert p.is_available() is True
def test_config_schema(self):
p = RetainDBMemoryProvider()
schema = p.get_config_schema()
assert len(schema) == 3
keys = [s["key"] for s in schema]
assert "api_key" in keys
assert "base_url" in keys
assert "project" in keys
def test_initialize_creates_client_and_queue(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
assert p._client is not None
assert p._queue is not None
assert p._session_id == "test-session"
p.shutdown()
def test_initialize_default_project(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
assert p._client.project == "default"
p.shutdown()
def test_initialize_explicit_project(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_PROJECT", "my-project")
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
assert p._client.project == "my-project"
p.shutdown()
def test_initialize_profile_project(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
profile_home = str(tmp_path / "profiles" / "coder")
p.initialize("test-session", hermes_home=profile_home)
assert p._client.project == "hermes-coder"
p.shutdown()
def test_initialize_seeds_soul_md(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
soul_path = tmp_path / ".hermes" / "SOUL.md"
soul_path.write_text("I am a helpful agent.")
with patch.object(RetainDBMemoryProvider, "_seed_soul") as mock_seed:
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
# Give thread time to start
time.sleep(0.5)
mock_seed.assert_called_once_with("I am a helpful agent.")
p.shutdown()
def test_system_prompt_block(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
block = p.system_prompt_block()
assert "RetainDB Memory" in block
assert "Active" in block
p.shutdown()
def test_handle_tool_call_not_initialized(self):
p = RetainDBMemoryProvider()
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
assert "error" in result
assert "not initialized" in result["error"]
def test_handle_tool_call_unknown_tool(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_nonexistent", {}))
assert result == {"error": "Unknown tool: retaindb_nonexistent"}
p.shutdown()
def test_dispatch_profile(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "get_profile", return_value={"memories": []}):
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
assert "memories" in result
p.shutdown()
def test_dispatch_search_requires_query(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_search", {}))
assert result == {"error": "query is required"}
p.shutdown()
def test_dispatch_search(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "search", return_value={"results": [{"content": "found"}]}):
result = json.loads(p.handle_tool_call("retaindb_search", {"query": "test"}))
assert "results" in result
p.shutdown()
def test_dispatch_search_top_k_capped(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "search") as mock_search:
mock_search.return_value = {"results": []}
p.handle_tool_call("retaindb_search", {"query": "test", "top_k": 100})
# top_k should be capped at 20
assert mock_search.call_args[1]["top_k"] == 20
p.shutdown()
def test_dispatch_remember(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}):
result = json.loads(p.handle_tool_call("retaindb_remember", {"content": "test fact"}))
assert result["id"] == "mem-1"
p.shutdown()
def test_dispatch_remember_requires_content(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_remember", {}))
assert result == {"error": "content is required"}
p.shutdown()
def test_dispatch_forget(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "delete_memory", return_value={"deleted": True}):
result = json.loads(p.handle_tool_call("retaindb_forget", {"memory_id": "mem-1"}))
assert result["deleted"] is True
p.shutdown()
def test_dispatch_forget_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_forget", {}))
assert result == {"error": "memory_id is required"}
p.shutdown()
def test_dispatch_context(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "query_context", return_value={"results": [{"content": "relevant"}]}), \
patch.object(p._client, "get_profile", return_value={"memories": []}):
result = json.loads(p.handle_tool_call("retaindb_context", {"query": "current task"}))
assert "context" in result
assert "raw" in result
p.shutdown()
def test_dispatch_file_list(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "list_files", return_value={"files": []}):
result = json.loads(p.handle_tool_call("retaindb_list_files", {}))
assert "files" in result
p.shutdown()
def test_dispatch_file_upload_missing_path(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_upload_file", {}))
assert "error" in result
def test_dispatch_file_upload_not_found(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_upload_file", {"local_path": "/nonexistent/file.txt"}))
assert "File not found" in result["error"]
p.shutdown()
def test_dispatch_file_read_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_read_file", {}))
assert result == {"error": "file_id is required"}
p.shutdown()
def test_dispatch_file_ingest_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_ingest_file", {}))
assert result == {"error": "file_id is required"}
p.shutdown()
def test_dispatch_file_delete_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_delete_file", {}))
assert result == {"error": "file_id is required"}
p.shutdown()
def test_handle_tool_call_wraps_exception(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "get_profile", side_effect=RuntimeError("API exploded")):
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
assert "API exploded" in result["error"]
p.shutdown()
# ===========================================================================
# Prefetch and thread management tests
# ===========================================================================
class TestPrefetch:
"""Test background prefetch and thread accumulation prevention."""
def _make_initialized_provider(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
return p
def test_queue_prefetch_skips_without_client(self):
p = RetainDBMemoryProvider()
p.queue_prefetch("test") # Should not raise
def test_prefetch_returns_empty_when_nothing_cached(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
result = p.prefetch("test")
assert result == ""
p.shutdown()
def test_prefetch_consumes_context_result(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
# Manually set the cached result
with p._lock:
p._context_result = "[RetainDB Context]\nProfile:\n- User likes tests"
result = p.prefetch("test")
assert "User likes tests" in result
# Should be consumed
assert p.prefetch("test") == ""
p.shutdown()
def test_prefetch_consumes_dialectic_result(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
with p._lock:
p._dialectic_result = "User is a software engineer who prefers Python."
result = p.prefetch("test")
assert "[RetainDB User Synthesis]" in result
assert "software engineer" in result
p.shutdown()
def test_prefetch_consumes_agent_model(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
with p._lock:
p._agent_model = {
"memory_count": 5,
"persona": "Helpful coding assistant",
"persistent_instructions": ["Be concise", "Use Python"],
"working_style": "Direct and efficient",
}
result = p.prefetch("test")
assert "[RetainDB Agent Self-Model]" in result
assert "Helpful coding assistant" in result
assert "Be concise" in result
assert "Direct and efficient" in result
p.shutdown()
def test_prefetch_skips_empty_agent_model(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
with p._lock:
p._agent_model = {"memory_count": 0}
result = p.prefetch("test")
assert "Agent Self-Model" not in result
p.shutdown()
def test_thread_accumulation_guard(self, tmp_path, monkeypatch):
"""Verify old prefetch threads are joined before new ones spawn."""
p = self._make_initialized_provider(tmp_path, monkeypatch)
# Mock the prefetch methods to be slow
with patch.object(p, "_prefetch_context", side_effect=lambda q: time.sleep(0.5)), \
patch.object(p, "_prefetch_dialectic", side_effect=lambda q: time.sleep(0.5)), \
patch.object(p, "_prefetch_agent_model", side_effect=lambda: time.sleep(0.5)):
p.queue_prefetch("query 1")
first_threads = list(p._prefetch_threads)
assert len(first_threads) == 3
# Call again — should join first batch before spawning new
p.queue_prefetch("query 2")
second_threads = list(p._prefetch_threads)
assert len(second_threads) == 3
# Should be different thread objects
for t in second_threads:
assert t not in first_threads
p.shutdown()
def test_reasoning_level_short(self):
assert RetainDBMemoryProvider._reasoning_level("hi") == "low"
def test_reasoning_level_medium(self):
assert RetainDBMemoryProvider._reasoning_level("x" * 200) == "medium"
def test_reasoning_level_long(self):
assert RetainDBMemoryProvider._reasoning_level("x" * 500) == "high"
# ===========================================================================
# sync_turn tests
# ===========================================================================
class TestSyncTurn:
"""Test turn synchronization via the write queue."""
def test_sync_turn_enqueues(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._queue, "enqueue") as mock_enqueue:
p.sync_turn("user msg", "assistant msg")
mock_enqueue.assert_called_once()
args = mock_enqueue.call_args[0]
assert args[0] == "default" # user_id
assert args[1] == "test-session" # session_id
msgs = args[2]
assert len(msgs) == 2
assert msgs[0]["role"] == "user"
assert msgs[1]["role"] == "assistant"
p.shutdown()
def test_sync_turn_skips_empty_user_content(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._queue, "enqueue") as mock_enqueue:
p.sync_turn("", "assistant msg")
mock_enqueue.assert_not_called()
p.shutdown()
# ===========================================================================
# on_memory_write hook tests
# ===========================================================================
class TestOnMemoryWrite:
"""Test the built-in memory mirror hook."""
def test_mirrors_add_action(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}) as mock_add:
p.on_memory_write("add", "user", "User prefers dark mode")
mock_add.assert_called_once()
assert mock_add.call_args[1]["memory_type"] == "preference"
p.shutdown()
def test_skips_non_add_action(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory") as mock_add:
p.on_memory_write("remove", "user", "something")
mock_add.assert_not_called()
p.shutdown()
def test_skips_empty_content(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory") as mock_add:
p.on_memory_write("add", "user", "")
mock_add.assert_not_called()
p.shutdown()
def test_memory_target_maps_to_type(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}) as mock_add:
p.on_memory_write("add", "memory", "Some env fact")
assert mock_add.call_args[1]["memory_type"] == "factual"
p.shutdown()
# ===========================================================================
# register() test
# ===========================================================================
class TestRegister:
def test_register_calls_register_memory_provider(self):
from plugins.memory.retaindb import register
ctx = MagicMock()
register(ctx)
ctx.register_memory_provider.assert_called_once()
arg = ctx.register_memory_provider.call_args[0][0]
assert isinstance(arg, RetainDBMemoryProvider)
tests/plugins/test_security_guidance_plugin.py
+332
View File
@@ -0,0 +1,332 @@
"""Tests for the security-guidance plugin.
Covers ``plugins/security-guidance/``:
* ``patterns.py`` data integrity — every rule has a ``RuleId``, the
fail-loud import assertion is wired.
* ``_scan_content`` — true positives (pickle.load, yaml.load, eval,
dangerouslySetInnerHTML, GitHub Actions workflow), true negatives
(.md skips Python rules, ``model.eval()`` doesn't trip eval),
path-only rules (``path_check``), content-only rules
(``path_filter``).
* Hooks — ``transform_tool_result`` appends a warning block in warn
mode and stays out of error results; ``pre_tool_call`` blocks
writes when ``SECURITY_GUIDANCE_BLOCK=1`` and stays silent
otherwise.
* Bundled-plugin discovery via ``PluginManager.discover_and_load``.
"""
import importlib.util
import sys
import types
from pathlib import Path
import pytest
@pytest.fixture(autouse=True)
def _isolate_env(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.delenv("SECURITY_GUIDANCE_BLOCK", raising=False)
monkeypatch.delenv("SECURITY_GUIDANCE_DISABLE", raising=False)
yield hermes_home
# ---------------------------------------------------------------------------
# Module loading
# ---------------------------------------------------------------------------
def _repo_root() -> Path:
return Path(__file__).resolve().parents[2]
def _load_patterns():
"""Import patterns.py in isolation (no plugin glue)."""
pat_path = _repo_root() / "plugins" / "security-guidance" / "patterns.py"
spec = importlib.util.spec_from_file_location(
"security_guidance_patterns_under_test", pat_path
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _load_plugin_init():
"""Import the plugin __init__.py with patterns.py as a sibling."""
plugin_dir = _repo_root() / "plugins" / "security-guidance"
if "hermes_plugins" not in sys.modules:
ns = types.ModuleType("hermes_plugins")
ns.__path__ = []
sys.modules["hermes_plugins"] = ns
spec = importlib.util.spec_from_file_location(
"hermes_plugins.security_guidance",
plugin_dir / "__init__.py",
submodule_search_locations=[str(plugin_dir)],
)
mod = importlib.util.module_from_spec(spec)
mod.__package__ = "hermes_plugins.security_guidance"
mod.__path__ = [str(plugin_dir)]
sys.modules["hermes_plugins.security_guidance"] = mod
spec.loader.exec_module(mod)
return mod
# ---------------------------------------------------------------------------
# patterns.py data integrity
# ---------------------------------------------------------------------------
class TestPatternsData:
def test_has_at_least_one_rule(self):
p = _load_patterns()
assert len(p.SECURITY_PATTERNS) >= 1
def test_every_rule_has_required_fields(self):
p = _load_patterns()
for rule in p.SECURITY_PATTERNS:
assert "ruleName" in rule
assert "reminder" in rule and rule["reminder"]
# At least one of substrings/regex/path_check must be present —
# otherwise the rule could never fire.
assert any(k in rule for k in ("substrings", "regex", "path_check")), rule
def test_rule_names_are_unique(self):
p = _load_patterns()
names = [r["ruleName"] for r in p.SECURITY_PATTERNS]
assert len(names) == len(set(names))
def test_rule_id_enum_in_sync(self):
# The upstream patterns.py asserts this at import time. If the
# set diverges, the import itself raises and this test fails.
p = _load_patterns()
rule_names = {r["ruleName"] for r in p.SECURITY_PATTERNS}
enum_names = set(p._RULE_NAME_TO_ID)
assert rule_names == enum_names
def test_rule_names_to_mask_packs_bits(self):
p = _load_patterns()
# PICKLE_DESERIALIZATION = 8, EVAL_INJECTION = 4 → bits 8 and 4 set.
mask = p.rule_names_to_mask({"pickle_deserialization", "eval_injection"})
assert mask & (1 << p.RuleId.PICKLE_DESERIALIZATION)
assert mask & (1 << p.RuleId.EVAL_INJECTION)
# ---------------------------------------------------------------------------
# _scan_content
# ---------------------------------------------------------------------------
class TestScanContent:
def test_pickle_load_in_py_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.py", "import pickle\nx = pickle.load(open('p.pkl', 'rb'))\n"
)
names = [n for n, _ in findings]
assert "pickle_deserialization" in names
def test_pickle_load_in_md_skipped_by_path_filter(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.md", "import pickle\nx = pickle.load(open('p.pkl', 'rb'))\n"
)
assert findings == []
def test_method_call_eval_does_not_trip(self):
"""model.eval() / redis.eval() / spec.eval() must not match eval_injection."""
mod = _load_plugin_init()
findings = mod._scan_content("/tmp/foo.py", "model.eval()\nout = model(x)\n")
assert "eval_injection" not in [n for n, _ in findings]
def test_bare_eval_in_py_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content("/tmp/foo.py", "result = eval(user_input)\n")
assert "eval_injection" in [n for n, _ in findings]
def test_subprocess_shell_true_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.py", "subprocess.run('ls ' + path, shell=True)\n"
)
assert "python_subprocess_shell" in [n for n, _ in findings]
def test_dangerously_set_inner_html_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.tsx", "<div dangerouslySetInnerHTML={{__html: x}} />"
)
assert "react_dangerously_set_html" in [n for n, _ in findings]
def test_github_workflow_path_check_fires_on_path_alone(self):
"""github_actions_workflow has no regex/substring — fires on path."""
mod = _load_plugin_init()
findings = mod._scan_content(
".github/workflows/test.yml", "name: CI\non: pull_request"
)
assert "github_actions_workflow" in [n for n, _ in findings]
def test_non_workflow_path_doesnt_trip_workflow_rule(self):
mod = _load_plugin_init()
findings = mod._scan_content("/tmp/foo.py", "name: CI")
assert "github_actions_workflow" not in [n for n, _ in findings]
def test_empty_content_returns_no_findings(self):
mod = _load_plugin_init()
assert mod._scan_content("/tmp/foo.py", "") == []
def test_huge_content_skipped(self):
mod = _load_plugin_init()
# 1 MB of content with a dangerous pattern at the end — scanner caps
# out at _MAX_SCAN_BYTES (256 KB), so this should return [].
big = "x" * (1024 * 1024) + "\npickle.load(open('p.pkl', 'rb'))\n"
assert mod._scan_content("/tmp/foo.py", big) == []
# ---------------------------------------------------------------------------
# Hooks
# ---------------------------------------------------------------------------
class TestTransformToolResultHook:
def test_warns_on_write_file_with_dangerous_content(self):
mod = _load_plugin_init()
args = {
"path": "/tmp/foo.py",
"content": "import pickle\nx = pickle.loads(b)\n",
}
result = mod._on_transform_tool_result(
tool_name="write_file",
args=args,
result='{"success": true, "bytes_written": 30}',
)
assert isinstance(result, str)
assert "Security guidance" in result
assert "pickle_deserialization" in result
# The original JSON should still be there at the start of the string.
assert result.startswith('{"success": true')
def test_no_warn_on_clean_content(self):
mod = _load_plugin_init()
args = {"path": "/tmp/foo.py", "content": "import json\nx = json.loads(b)\n"}
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"success": true}'
)
is None
)
def test_no_warn_when_result_is_error(self):
mod = _load_plugin_init()
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
# When the tool itself errored, we don't pile a security warning on
# top — the model has bigger problems to solve.
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"error": "boom"}'
)
is None
)
def test_patch_tool_new_string_scanned(self):
mod = _load_plugin_init()
args = {
"path": "/tmp/foo.py",
"old_string": "x = 1",
"new_string": "x = eval(user_input)",
}
result = mod._on_transform_tool_result(
tool_name="patch", args=args, result='{"success": true}'
)
assert isinstance(result, str)
assert "eval_injection" in result
def test_untargeted_tool_skipped(self):
mod = _load_plugin_init()
# The plugin only scans write_file/patch/skill_manage. terminal output
# should pass through untouched.
args = {"command": "echo pickle.load"}
assert (
mod._on_transform_tool_result(
tool_name="terminal", args=args, result='{"output": "pickle.load"}'
)
is None
)
def test_disable_kill_switch(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_DISABLE", "1")
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"ok": true}'
)
is None
)
def test_block_mode_makes_transform_hook_quiet(self, monkeypatch):
"""In block mode, pre_tool_call handles the warning; the transform
hook stays silent so we don't double-emit."""
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"ok": true}'
)
is None
)
class TestPreToolCallHook:
def test_no_block_in_warn_mode(self):
mod = _load_plugin_init()
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
assert mod._on_pre_tool_call(tool_name="write_file", args=args) is None
def test_blocks_in_block_mode_on_dangerous_pattern(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
out = mod._on_pre_tool_call(tool_name="write_file", args=args)
assert isinstance(out, dict)
assert out["action"] == "block"
assert "pickle_deserialization" in out["message"]
assert "SECURITY_GUIDANCE_BLOCK" in out["message"] # tells user how to disable
def test_no_block_in_block_mode_on_clean_content(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"path": "/tmp/foo.py", "content": "import json\n"}
assert mod._on_pre_tool_call(tool_name="write_file", args=args) is None
def test_untargeted_tool_skipped(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"command": "echo pickle.load(f)"}
assert mod._on_pre_tool_call(tool_name="terminal", args=args) is None
# ---------------------------------------------------------------------------
# Bundled-plugin discovery
# ---------------------------------------------------------------------------
class TestPluginDiscovery:
def test_loads_via_plugin_manager(self, _isolate_env, monkeypatch):
"""End-to-end: enable in config.yaml and verify the PluginManager
picks it up via the standard discovery path."""
import yaml
config = {"plugins": {"enabled": ["security-guidance"]}}
(_isolate_env / "config.yaml").write_text(yaml.safe_dump(config))
# Wipe any cached plugin state from earlier tests in this worker.
for k in list(sys.modules):
if k.startswith(("hermes_plugins", "hermes_cli.plugins")):
del sys.modules[k]
from hermes_cli.plugins import _ensure_plugins_discovered
mgr = _ensure_plugins_discovered(force=True)
loaded = set()
if hasattr(mgr, "_plugins"):
loaded = set(mgr._plugins.keys())
assert "security-guidance" in loaded
tests/plugins/test_teams_pipeline_plugin.py
+467
View File
@@ -0,0 +1,467 @@
"""Tests for the Teams pipeline plugin package."""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from pathlib import Path
import pytest
from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest
from gateway.config import GatewayConfig, Platform, PlatformConfig
from plugins.teams_pipeline import register
from plugins.teams_pipeline.pipeline import TeamsMeetingPipeline
from plugins.teams_pipeline.store import TeamsPipelineStore
from plugins.teams_pipeline.models import MeetingArtifact
class FakeGraphClient:
def __init__(self) -> None:
self.downloaded = False
async def _transcript_meeting_resolver(client, *, meeting_id=None, join_web_url=None, tenant_id=None):
from plugins.teams_pipeline.models import TeamsMeetingRef
return TeamsMeetingRef(
meeting_id=str(meeting_id),
tenant_id=tenant_id,
metadata={"subject": "Weekly Sync", "participants": [{"displayName": "Ada"}]},
)
async def _no_call_record(*args, **kwargs):
return None
def test_register_adds_cli_only():
mgr = PluginManager()
manifest = PluginManifest(name="teams_pipeline")
ctx = PluginContext(manifest, mgr)
register(ctx)
assert "teams-pipeline" in mgr._cli_commands
entry = mgr._cli_commands["teams-pipeline"]
assert entry["plugin"] == "teams_pipeline"
assert callable(entry["setup_fn"])
assert callable(entry["handler_fn"])
def test_runtime_config_uses_existing_teams_platform_settings():
from plugins.teams_pipeline.runtime import build_pipeline_runtime_config
gateway_config = GatewayConfig(
platforms={
Platform("teams"): PlatformConfig(
enabled=True,
extra={
"delivery_mode": "graph",
"team_id": "team-1",
"channel_id": "channel-1",
"meeting_pipeline": {
"transcript_min_chars": 120,
"notion": {"enabled": True, "database_id": "db-1"},
},
},
)
}
)
runtime_config = build_pipeline_runtime_config(gateway_config)
assert runtime_config["transcript_min_chars"] == 120
assert runtime_config["notion"]["database_id"] == "db-1"
assert runtime_config["teams_delivery"] == {
"enabled": True,
"mode": "graph",
"team_id": "team-1",
"channel_id": "channel-1",
}
def test_build_pipeline_runtime_reuses_existing_teams_adapter_surface(monkeypatch, tmp_path):
from plugins.teams_pipeline import runtime as runtime_module
class FakeWriter:
def __init__(self, platform_config=None, **kwargs) -> None:
self.platform_config = platform_config
monkeypatch.setattr(runtime_module, "build_graph_client", lambda: object())
monkeypatch.setattr(runtime_module, "resolve_teams_pipeline_store_path", lambda: tmp_path / "teams-store.json")
monkeypatch.setattr("plugins.platforms.teams.adapter.TeamsSummaryWriter", FakeWriter)
gateway = SimpleNamespace(
config=GatewayConfig(
platforms={
Platform("teams"): PlatformConfig(
enabled=True,
extra={
"delivery_mode": "incoming_webhook",
"incoming_webhook_url": "https://example.com/hook",
},
)
}
)
)
runtime = runtime_module.build_pipeline_runtime(gateway)
assert isinstance(runtime.teams_sender, FakeWriter)
assert runtime.teams_sender.platform_config is gateway.config.platforms[Platform("teams")]
@pytest.mark.anyio
async def test_bind_gateway_runtime_attaches_scheduler(monkeypatch, tmp_path):
from plugins.teams_pipeline import runtime as runtime_module
class FakeAdapter:
def __init__(self) -> None:
self.scheduler = None
def set_notification_scheduler(self, scheduler) -> None:
self.scheduler = scheduler
class FakePipeline:
def __init__(self) -> None:
self.notifications = []
async def run_notification(self, notification):
self.notifications.append(notification)
adapter = FakeAdapter()
pipeline = FakePipeline()
gateway = SimpleNamespace(
adapters={Platform.MSGRAPH_WEBHOOK: adapter},
config=GatewayConfig(platforms={}),
_teams_pipeline_runtime=None,
_teams_pipeline_runtime_error=None,
)
monkeypatch.setattr(runtime_module, "build_pipeline_runtime", lambda gateway_runner: pipeline)
bound = runtime_module.bind_gateway_runtime(gateway)
assert bound is True
assert gateway._teams_pipeline_runtime is pipeline
assert callable(adapter.scheduler)
notification = {"id": "notif-1"}
await adapter.scheduler(notification, object())
assert pipeline.notifications == [notification]
@pytest.mark.anyio
async def test_bind_gateway_runtime_drops_notifications_when_unavailable(monkeypatch):
from plugins.teams_pipeline import runtime as runtime_module
from tools.microsoft_graph_auth import MicrosoftGraphConfigError
class FakeAdapter:
def __init__(self) -> None:
self.scheduler = None
def set_notification_scheduler(self, scheduler) -> None:
self.scheduler = scheduler
adapter = FakeAdapter()
gateway = SimpleNamespace(
adapters={Platform.MSGRAPH_WEBHOOK: adapter},
config=GatewayConfig(platforms={}),
_teams_pipeline_runtime=None,
_teams_pipeline_runtime_error=None,
)
def _raise(_gateway_runner):
raise MicrosoftGraphConfigError("missing graph env")
monkeypatch.setattr(runtime_module, "build_pipeline_runtime", _raise)
bound = runtime_module.bind_gateway_runtime(gateway)
assert bound is False
assert "missing graph env" in gateway._teams_pipeline_runtime_error
assert callable(adapter.scheduler)
await adapter.scheduler({"id": "notif-2"}, object())
def test_store_persists_subscription_event_and_job_state(tmp_path):
store_path = tmp_path / "teams-store.json"
store = TeamsPipelineStore(store_path)
store.upsert_subscription(
"sub-1",
{"client_state": "abc", "resource": "communications/onlineMeetings"},
)
store.record_event_timestamp("evt-1", "2026-05-03T19:30:00Z")
store.upsert_job("job-1", {"status": "received", "event_id": "evt-1"})
store.upsert_sink_record("notion:meeting-1", {"page_id": "page-1"})
reloaded = TeamsPipelineStore(store_path)
subscription = reloaded.get_subscription("sub-1")
job = reloaded.get_job("job-1")
sink = reloaded.get_sink_record("notion:meeting-1")
assert subscription is not None
assert subscription["subscription_id"] == "sub-1"
assert subscription["client_state"] == "abc"
assert reloaded.get_event_timestamp("evt-1") == "2026-05-03T19:30:00Z"
assert job is not None
assert job["status"] == "received"
assert sink is not None
assert sink["page_id"] == "page-1"
def test_store_notification_receipts_are_idempotent(tmp_path):
store = TeamsPipelineStore(tmp_path / "teams-store.json")
notification = {
"subscriptionId": "sub-1",
"resource": "communications/onlineMeetings/meeting-1",
"changeType": "updated",
}
receipt_key = TeamsPipelineStore.build_notification_receipt_key(notification)
assert store.record_notification_receipt(receipt_key, notification) is True
assert store.record_notification_receipt(receipt_key, notification) is False
assert store.has_notification_receipt(receipt_key) is True
reloaded = TeamsPipelineStore(tmp_path / "teams-store.json")
assert reloaded.has_notification_receipt(receipt_key) is True
@pytest.mark.anyio
class TestTeamsMeetingPipeline:
async def test_transcript_first_path_persists_state_and_skips_recording(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
async def _fetch_transcript(client, meeting_ref):
return (
MeetingArtifact(artifact_type="transcript", artifact_id="tx-1", display_name="meeting.vtt"),
"Action: Send draft by Friday.\nDecision: Ship the transcript-first path.\nDetailed transcript content.",
)
async def _call_record(client, meeting_ref, *, call_record_id=None, allow_permission_errors=True):
return MeetingArtifact(
artifact_type="call_record",
artifact_id="call-1",
metadata={"metrics": {"participant_count": 4}},
)
async def _summarize(**kwargs):
return pipeline_module.TeamsMeetingSummaryPayload(
meeting_ref=kwargs["resolved_meeting"],
title="Weekly Sync",
transcript_text=kwargs["transcript_text"],
summary="Short summary",
key_decisions=["Ship the transcript-first path."],
action_items=["Send draft by Friday."],
risks=["Timeline risk."],
confidence="high",
confidence_notes="Transcript available.",
source_artifacts=kwargs["artifacts"],
)
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _fetch_transcript)
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _call_record)
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={"transcript_min_chars": 20},
summarize_fn=_summarize,
)
job = await pipeline.run_notification(
{
"id": "notif-1",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-123",
"resourceData": {"id": "meeting-123"},
}
)
assert job.status == "completed"
assert job.selected_artifact_strategy == "transcript_first"
assert job.summary_payload is not None
assert job.summary_payload.summary == "Short summary"
stored = store.get_job(job.job_id)
assert stored is not None
assert stored["status"] == "completed"
async def test_recording_fallback_uses_stt_and_updates_sink_records(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
async def _no_transcript(client, meeting_ref):
return None, None
async def _recordings(client, meeting_ref):
return [
MeetingArtifact(
artifact_type="recording",
artifact_id="rec-1",
display_name="recording.mp4",
download_url="https://files.example/recording.mp4",
)
]
async def _download(client, meeting_ref, recording, destination):
target = Path(destination)
target.write_bytes(b"video-bytes")
return {"path": str(target), "size_bytes": 11, "content_type": "video/mp4"}
async def _prepare_audio(self, recording_path):
audio_path = recording_path.with_suffix(".wav")
audio_path.write_bytes(b"audio-bytes")
return audio_path
def _transcribe(file_path, model):
return {"success": True, "transcript": "Action: Follow up with Legal.\nRisk: Budget approval pending.", "provider": "local"}
async def _summarize(**kwargs):
return pipeline_module.TeamsMeetingSummaryPayload(
meeting_ref=kwargs["resolved_meeting"],
title="Weekly Sync",
transcript_text=kwargs["transcript_text"],
summary="Fallback summary",
key_decisions=[],
action_items=["Follow up with Legal."],
risks=["Budget approval pending."],
confidence="medium",
confidence_notes="Generated from STT fallback.",
source_artifacts=kwargs["artifacts"],
)
class FakeNotionWriter:
async def write_summary(self, payload, config, existing_record=None):
return {"page_id": existing_record.get("page_id") if existing_record else "page-1", "url": "https://notion.so/page-1"}
async def _teams_sender(payload, config, existing_record=None):
return {"message_id": existing_record.get("message_id") if existing_record else "msg-1"}
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _no_transcript)
monkeypatch.setattr(pipeline_module, "list_recording_artifacts", _recordings)
monkeypatch.setattr(pipeline_module, "download_recording_artifact", _download)
monkeypatch.setattr(pipeline_module.TeamsMeetingPipeline, "_prepare_audio_path", _prepare_audio)
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _no_call_record)
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={
"notion": {"enabled": True, "database_id": "db-1"},
"teams_delivery": {"enabled": True, "channel_id": "channel-1"},
},
transcribe_fn=_transcribe,
summarize_fn=_summarize,
notion_writer=FakeNotionWriter(),
teams_sender=_teams_sender,
)
job = await pipeline.run_notification(
{
"id": "notif-2",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-456",
"resourceData": {"id": "meeting-456"},
}
)
assert job.status == "completed"
assert job.selected_artifact_strategy == "recording_stt_fallback"
assert job.summary_payload is not None
assert job.summary_payload.summary == "Fallback summary"
notion_record = store.get_sink_record("notion:meeting-456")
teams_record = store.get_sink_record("teams:meeting-456")
assert notion_record is not None
assert notion_record["page_id"] == "page-1"
assert teams_record is not None
assert teams_record["message_id"] == "msg-1"
async def test_missing_transcript_and_recording_schedules_retry(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", lambda *a, **kw: asyncio.sleep(0, result=(None, None)))
monkeypatch.setattr(pipeline_module, "list_recording_artifacts", lambda *a, **kw: asyncio.sleep(0, result=[]))
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={},
summarize_fn=lambda **kwargs: asyncio.sleep(0, result=None),
)
job = await pipeline.run_notification(
{
"id": "notif-3",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-789",
"resourceData": {"id": "meeting-789"},
}
)
assert job.status == "retry_scheduled"
assert job.error_info["retryable"] is True
assert "Recording unavailable" in job.error_info["message"]
async def test_duplicate_notification_reuses_completed_job(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
async def _fetch_transcript(client, meeting_ref):
return (
MeetingArtifact(artifact_type="transcript", artifact_id="tx-dup", display_name="meeting.vtt"),
"Decision: Keep duplicate notifications idempotent.\nAction: Verify the cached job is reused.",
)
summarize_calls = 0
async def _summarize(**kwargs):
nonlocal summarize_calls
summarize_calls += 1
return pipeline_module.TeamsMeetingSummaryPayload(
meeting_ref=kwargs["resolved_meeting"],
title="Weekly Sync",
transcript_text=kwargs["transcript_text"],
summary="Duplicate-safe summary",
key_decisions=["Keep duplicate notifications idempotent."],
action_items=["Verify the cached job is reused."],
confidence="high",
confidence_notes="Transcript available.",
source_artifacts=kwargs["artifacts"],
)
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _fetch_transcript)
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _no_call_record)
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={"transcript_min_chars": 20},
summarize_fn=_summarize,
)
notification = {
"id": "notif-dup",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-dup",
"resourceData": {"id": "meeting-dup"},
}
first_job = await pipeline.run_notification(notification)
second_job = await pipeline.run_notification(notification)
assert first_job.status == "completed"
assert second_job.status == "completed"
assert second_job.job_id == first_job.job_id
assert summarize_calls == 1
assert len(store.list_jobs()) == 1
receipt_key = TeamsPipelineStore.build_notification_receipt_key(notification)
assert store.has_notification_receipt(receipt_key) is True
tests/plugins/transcription/__init__.py
View File
tests/plugins/transcription/check_parity_vs_main.py
+431
View File
@@ -0,0 +1,431 @@
"""Behavior-parity check for the STT plugin hook + command-provider registry.
Spawns one subprocess per (version, scenario) cell — pinned to either
``origin/main`` (no plugin hook, no STT command-provider registry; only
the legacy ``HERMES_LOCAL_STT_COMMAND`` escape hatch exists) or this PR's
worktree (both new surfaces present).
Each subprocess clears all STT-related env vars + writes a
``config.yaml``, then asks the dispatcher how it would route a
``transcribe_audio`` call. The emitted shape tuple is::
{dispatch_kind, provider_name, success}
Where ``dispatch_kind`` ∈
``{"builtin_local", "builtin_groq", "builtin_openai", ...,
"plugin", "plugin_unavailable", "command_provider",
"no_provider_error", "stt_disabled"}``.
Acceptable diffs:
- ``no_provider_error → plugin`` for the ``plugin-installed`` scenario.
- ``no_provider_error → plugin_unavailable`` for the
``plugin-installed-unavailable`` scenario (PR returns the cleaner
unavailability envelope instead of the generic auto-detect error).
- ``no_provider_error → command_provider`` for the
``command-provider-installed`` scenario (registry shipped with this PR).
- ``no_provider_error → command_provider`` for
``command-vs-plugin-same-name`` (command wins precedence, same as TTS).
Run from the PR worktree::
python tests/plugins/transcription/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "transcription_tools.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "transcription_tools.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "transcription_tools.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear STT-related env so dispatch decisions are config-driven.
for k in (
"GROQ_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY",
"MISTRAL_API_KEY", "XAI_API_KEY",
"HERMES_LOCAL_STT_COMMAND",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
plugin_register = sys.argv[4] # "yes" to register a fake plugin
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached from prior runs.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
# Try importing transcription_registry — only exists on PR side.
have_plugin_hook = False
try:
from agent import transcription_registry
from agent.transcription_provider import TranscriptionProvider
have_plugin_hook = True
if plugin_register == "yes":
class _FakeProvider(TranscriptionProvider):
@property
def name(self): return "openrouter"
def transcribe(self, file_path, **kw):
return {"success": True, "transcript": "PLUGIN: openrouter transcript", "provider": "openrouter"}
transcription_registry._reset_for_tests()
transcription_registry.register_provider(_FakeProvider())
elif plugin_register == "unavailable":
class _UnavailablePlugin(TranscriptionProvider):
@property
def name(self): return "openrouter"
def is_available(self): return False
def transcribe(self, file_path, **kw):
return {"success": True, "transcript": "should not run"}
transcription_registry._reset_for_tests()
transcription_registry.register_provider(_UnavailablePlugin())
except ImportError:
pass
import tools.transcription_tools as tt
# Use a real (but empty) audio file so _validate_audio_file passes.
audio_path = os.path.join(home, "audio.ogg")
with open(audio_path, "wb") as f:
# Minimal-ish OGG-shaped bytes so the size check passes.
f.write(b"OggS" + b"\x00" * 1024)
# Patch _transcribe_* so the test doesn't actually try cloud APIs.
# We're testing dispatch, not the underlying transcription.
def _stub(file_path, model_name=None):
return {"success": True, "transcript": "stub from " + sys._getframe().f_code.co_name.replace("_stub_", ""),
"provider": sys._getframe().f_code.co_name.replace("_stub_", "")}
# Stub each built-in to a marker so we can identify the branch.
class _Stub:
def __init__(self, name):
self.name = name
def __call__(self, file_path, model_name=None):
return {"success": True, "transcript": "stub", "provider": self.name}
tt._transcribe_local = _Stub("local")
tt._transcribe_local_command = _Stub("local_command")
tt._transcribe_groq = _Stub("groq")
tt._transcribe_openai = _Stub("openai")
tt._transcribe_mistral = _Stub("mistral")
tt._transcribe_xai = _Stub("xai")
# Force _get_provider to honor the explicit config since we don't have
# real creds. The provider-resolution gates check _HAS_OPENAI /
# _HAS_FASTER_WHISPER which we can't easily set, so we just patch
# _get_provider to return whatever the config says.
stt_cfg = tt._load_stt_config()
explicit = stt_cfg.get("provider")
if explicit:
# Bypass the gating for test purposes — _get_provider would
# otherwise return "none" when the dependency isn't installed.
original_get = tt._get_provider
def _patched(cfg):
if not tt.is_stt_enabled(cfg):
return "none"
return cfg.get("provider", "none")
tt._get_provider = _patched
try:
result = tt.transcribe_audio(audio_path)
except Exception as exc:
shape = {"dispatch_kind": "exception", "provider_name": None, "success": False,
"error_text": repr(exc)}
print(json.dumps(shape))
sys.exit(0)
dispatch_kind = "unknown"
provider_name = result.get("provider") if isinstance(result, dict) else None
success = result.get("success", False) if isinstance(result, dict) else False
error_text = result.get("error", "") if isinstance(result, dict) else ""
if not success and "STT is disabled" in error_text:
dispatch_kind = "stt_disabled"
elif not success and "is not available" in error_text:
dispatch_kind = "plugin_unavailable"
elif not success and "No STT provider" in error_text:
dispatch_kind = "no_provider_error"
elif provider_name in ("local", "local_command", "groq", "openai", "mistral", "xai"):
dispatch_kind = "builtin_" + provider_name
elif success and isinstance(result, dict) and result.get("transcript", "").startswith("CMD:"):
# Command-provider scenarios below emit transcripts prefixed with "CMD:"
# so the harness can distinguish command-provider dispatch from a
# plugin dispatch even when they share a provider name.
dispatch_kind = "command_provider"
elif success and isinstance(result, dict) and result.get("transcript", "").startswith("PLUGIN:"):
dispatch_kind = "plugin"
elif success and provider_name and provider_name not in ("local", "local_command", "groq", "openai", "mistral", "xai"):
dispatch_kind = "plugin"
else:
dispatch_kind = "other"
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"success": success,
}
print(json.dumps(shape))
"""
def _cmd_yaml(provider_name: str, transcript: str) -> str:
"""Build a YAML snippet for an stt.providers.<name>: type: command entry.
Produces a shell command that writes ``transcript`` to {output_path}.
Backslashes in the venv python path are doubled for YAML, and the
inner double quotes around the python -c payload are YAML-escaped.
Keeps the test scenarios readable.
"""
interp = sys.executable.replace("\\", "\\\\")
# Inside the YAML double-quoted string, we use single quotes around
# the python -c body so we don't have to YAML-escape inner double
# quotes. Single quotes inside the body are not needed; the body uses
# double quotes for module references and string literals.
payload = (
f"import sys; open(sys.argv[1], 'w').write('{transcript}')"
)
command = f'{interp} -c "{payload}" {{output_path}}'
# YAML-escape: double-quote the whole thing, escape inner " and \.
yaml_escaped = command.replace("\\", "\\\\").replace('"', '\\"')
return (
"stt:\n"
f" provider: {provider_name}\n"
" providers:\n"
f" {provider_name}:\n"
" type: command\n"
f' command: "{yaml_escaped}"\n'
)
SCENARIOS: list[tuple[str, str, dict[str, str], str]] = [
# (label, config.yaml body, scenario_env, plugin_register)
("stt-disabled", "stt:\n enabled: false\n", {}, "no"),
("explicit-groq", "stt:\n provider: groq\n", {}, "no"),
("explicit-openai", "stt:\n provider: openai\n", {}, "no"),
("explicit-local", "stt:\n provider: local\n", {}, "no"),
("explicit-xai", "stt:\n provider: xai\n", {}, "no"),
# Mistral is quarantined → _get_provider returns "none" today, hence no_provider_error.
("explicit-mistral-quarantine", "stt:\n provider: mistral\n", {}, "no"),
# Unknown name + no plugin → both: no_provider_error
("unknown-no-plugin", "stt:\n provider: openrouter\n", {}, "no"),
# Unknown name + plugin installed → main: no_provider_error, PR: plugin
("plugin-installed", "stt:\n provider: openrouter\n", {}, "yes"),
# Unknown name + plugin reports unavailable → main: no_provider_error,
# PR: plugin_unavailable (cleaner envelope, names the plugin)
("plugin-installed-unavailable", "stt:\n provider: openrouter\n", {}, "unavailable"),
# Built-in name + plugin tries to shadow → both: built-in
("explicit-openai-with-plugin-registered", "stt:\n provider: openai\n", {}, "yes"),
# NEW (this PR): stt.providers.<name>: type: command registry.
# Provider name "fake-cli" + transcript prefixed "CMD:" so dispatch_kind
# detection routes it to "command_provider". On main (no registry),
# this falls through to no_provider_error.
(
"command-provider-installed",
_cmd_yaml("fake-cli", "CMD: fake-cli transcript"),
{},
"no",
),
# NEW (this PR): same name registered as BOTH a command provider and
# a plugin under "openrouter". Command must win (config more local
# than plugin install). The plugin emits "PLUGIN:..." — assertion is
# that the transcript is "CMD:...", proving command-wins precedence.
(
"command-vs-plugin-same-name",
_cmd_yaml("openrouter", "CMD: openrouter via command wins"),
{},
"yes", # also register a plugin under "openrouter" — must NOT fire
),
# NEW (this PR): built-in name with a command provider declared under
# it → built-in still wins (built-in elif chain has precedence).
# The command would write "CMD: HIJACK" if it fired — assertion is
# that built-in OpenAI dispatch fires instead.
(
"explicit-openai-with-command-shadow",
_cmd_yaml("openai", "CMD: HIJACK"),
{},
"no",
),
]
# Subprocesses reset the registry between runs via ``_reset_for_tests`` so
# registrations from earlier scenarios don't leak. The command-provider
# scenarios also work on origin/main — the subprocess just executes the
# native dispatch path, which falls through to "no_provider_error" because
# main has no registry for stt.providers.<name>.
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict, plugin_register: str) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / "venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
plugin_register,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
return {
"dispatch_kind": shape.get("dispatch_kind"),
"success": shape.get("success"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env, plugin_register in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env, plugin_register)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env, plugin_register)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "plugin-installed" returns no_provider_error (no
# plugin hook); on PR, plugin dispatches. Same shape for
# "plugin-installed-unavailable" but PR returns the cleaner
# plugin_unavailable envelope. The new command-provider scenarios
# also intentionally diff against main (which has no stt.providers
# registry yet).
no_provider_to_plugin = (
main_reduced.get("dispatch_kind") == "no_provider_error"
and pr_reduced.get("dispatch_kind") == "plugin"
and label == "plugin-installed"
)
no_provider_to_unavailable = (
main_reduced.get("dispatch_kind") == "no_provider_error"
and pr_reduced.get("dispatch_kind") == "plugin_unavailable"
and label == "plugin-installed-unavailable"
)
no_provider_to_command = (
main_reduced.get("dispatch_kind") == "no_provider_error"
and pr_reduced.get("dispatch_kind") == "command_provider"
and label in {"command-provider-installed", "command-vs-plugin-same-name"}
)
if no_provider_to_plugin:
print(f" [DIFF] {label}: no_provider_error → plugin — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
elif no_provider_to_unavailable:
print(f" [DIFF] {label}: no_provider_error → plugin_unavailable — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
elif no_provider_to_command:
print(f" [DIFF] {label}: no_provider_error → command_provider — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"no_provider_error → plugin dispatch when a plugin is registered."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
tests/plugins/tts/__init__.py
View File
tests/plugins/tts/check_parity_vs_main.py
+328
View File
@@ -0,0 +1,328 @@
"""Behavior-parity check for the TTS plugin hook (issue #30398).
Spawns one subprocess per (version, scenario) cell — pinned to either
``origin/main`` (no plugin hook; ``tts.provider: cartesia`` falls
through to the Edge TTS default branch) or this PR's worktree (plugin
hook present; same config routes through the plugin registry when a
plugin is registered).
Each subprocess clears all TTS-related env vars + writes a
``config.yaml``, then resolves how the dispatcher would route a
``text_to_speech`` call. The emitted shape tuple is::
{dispatch_kind, provider_name, voice_compat}
Where ``dispatch_kind`` ∈
``{"builtin_edge", "builtin_openai", "builtin_elevenlabs", ...,
"command", "plugin", "fallback_edge", "error"}``:
* ``builtin_<name>`` — config selects a built-in handler that exists
on both main and PR (no diff expected)
* ``command`` — config selects a ``tts.providers.<name>: type: command``
entry (PR #17843; no diff expected)
* ``plugin`` — config selects a plugin-registered provider (PR only)
* ``fallback_edge`` — config selects an unknown name with no matching
plugin or command entry → Edge TTS default fallback
* ``error`` — explicit fatal error (e.g. mistral quarantine)
The parent process diffs the reduced shape per scenario. The only
acceptable diff is ``fallback_edge → plugin`` for the
``unknown-name-with-plugin-installed`` scenario — everything else is
a regression.
Run from the PR worktree (it auto-resolves ``MAIN_DIR`` from the parent
of the worktree directory, or falls back to a sibling
``hermes-agent-main`` checkout)::
python tests/plugins/tts/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "tts_tool.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "tts_tool.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "tts_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
# The subprocess script — runs INSIDE either the main checkout or PR
# checkout, so the import paths resolve to the version of the code
# under test. We never call the real ``text_to_speech_tool`` because
# that would require audio synthesis; instead we ask the resolution
# layer what it WOULD do.
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear TTS-related env so dispatch decisions are config-driven.
for k in (
"ELEVENLABS_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY",
"MINIMAX_API_KEY", "XAI_API_KEY", "GEMINI_API_KEY",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
plugin_register = sys.argv[4] # "yes" to register a fake plugin
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached from prior runs.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
# Try importing tts_registry — only exists on PR side.
have_plugin_hook = False
try:
from agent import tts_registry
from agent.tts_provider import TTSProvider
have_plugin_hook = True
if plugin_register == "yes":
class _FakeProvider(TTSProvider):
@property
def name(self): return "cartesia"
def synthesize(self, text, output_path, **kw):
return output_path
tts_registry._reset_for_tests()
tts_registry.register_provider(_FakeProvider())
except ImportError:
pass
import tools.tts_tool as tts_tool
# Read the config the same way text_to_speech_tool() does.
tts_config = tts_tool._load_tts_config()
provider = tts_tool._get_provider(tts_config)
dispatch_kind = None
provider_name = provider
voice_compat = False
error_text = None
try:
# Mistral is the one branch that returns a fatal error.
if provider == "mistral":
dispatch_kind = "error"
error_text = "mistral quarantine"
elif tts_tool._resolve_command_provider_config(provider, tts_config) is not None:
dispatch_kind = "command"
elif have_plugin_hook and provider not in tts_tool.BUILTIN_TTS_PROVIDERS:
# On PR side: check plugin dispatch.
plugin_path = tts_tool._dispatch_to_plugin_provider(
"test", os.path.join(home, "out.mp3"), provider, tts_config,
)
if plugin_path is not None:
dispatch_kind = "plugin"
voice_compat = tts_tool._plugin_provider_is_voice_compatible(provider)
else:
# Falls through to Edge TTS default on the PR side too.
dispatch_kind = "fallback_edge"
elif provider in tts_tool.BUILTIN_TTS_PROVIDERS:
dispatch_kind = "builtin_" + provider
else:
# On main side: unknown names fall through to Edge default.
dispatch_kind = "fallback_edge"
except Exception as exc:
dispatch_kind = "exception"
error_text = repr(exc)
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"voice_compat": bool(voice_compat),
"error_present": error_text is not None,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str], str]] = [
# (label, config.yaml body, scenario_env, plugin_register)
# Scenario 1: unset tts.provider → both: Edge default
("unset-defaults-to-edge", "", {}, "no"),
# Scenario 2: built-in name → both: that built-in
("explicit-edge", "tts:\n provider: edge\n", {}, "no"),
("explicit-openai", "tts:\n provider: openai\n", {}, "no"),
("explicit-elevenlabs", "tts:\n provider: elevenlabs\n", {}, "no"),
# Scenario 3: command-type provider → both: command dispatch
(
"command-provider",
"tts:\n provider: my-piper\n providers:\n my-piper:\n type: command\n command: 'piper -m model.onnx -f {output_path} < {input_path}'\n",
{},
"no",
),
# Scenario 4: unknown name with NO plugin installed → both: fallback to Edge
("unknown-no-plugin", "tts:\n provider: cartesia\n", {}, "no"),
# Scenario 5: unknown name WITH plugin installed
# main: fallback_edge (no plugin hook exists)
# PR: plugin (cartesia)
# This is the ONLY acceptable diff in the harness.
("plugin-installed", "tts:\n provider: cartesia\n", {}, "yes"),
# Scenario 6: built-in name + plugin tries to shadow → both: built-in
# The plugin registers under name "cartesia", not "edge", so this is
# effectively the same as scenario 2 — but we exercise the with-plugin
# path to ensure the built-in branch still takes priority.
("explicit-edge-with-plugin-registered", "tts:\n provider: edge\n", {}, "yes"),
# Scenario 7: mistral quarantine — both surface the explicit error
("mistral-quarantine", "tts:\n provider: mistral\n", {}, "no"),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict, plugin_register: str) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / "venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
plugin_register,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
"""Reduce to the parts that matter for user-visible parity."""
return {
"dispatch_kind": shape.get("dispatch_kind"),
"provider_name": shape.get("provider_name"),
"error_present": shape.get("error_present"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env, plugin_register in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env, plugin_register)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env, plugin_register)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "plugin-installed" scenario returns fallback_edge
# (no plugin hook); on PR, it routes to the plugin. That's the
# only acceptable diff.
fallback_to_plugin = (
main_reduced.get("dispatch_kind") == "fallback_edge"
and pr_reduced.get("dispatch_kind") == "plugin"
and label == "plugin-installed"
)
if fallback_to_plugin:
print(f" [DIFF] {label}: fallback_edge → plugin — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"fallback_edge → plugin dispatch when a plugin is registered."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
tests/plugins/video_gen/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Make tests/plugins/video_gen a package."""
tests/plugins/video_gen/test_fal_plugin.py
+342
View File
@@ -0,0 +1,342 @@
"""Tests for the FAL video gen plugin — family routing, payload shape."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
def test_fal_provider_registers():
from plugins.video_gen.fal import FALVideoGenProvider, DEFAULT_MODEL
provider = FALVideoGenProvider()
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("fal") is provider
assert provider.display_name == "FAL"
# DEFAULT_MODEL is the cheap-tier default
assert provider.default_model() == DEFAULT_MODEL
assert DEFAULT_MODEL in {"pixverse-v6", "ltx-2.3"}
def test_fal_family_catalog():
"""Each family declares both endpoints. The catalog covers the
cheap + premium tiers Teknium listed."""
from plugins.video_gen.fal import FAL_FAMILIES
expected = {
# cheap
"ltx-2.3", "pixverse-v6",
# premium
"veo3.1", "seedance-2.0", "kling-v3-4k", "happy-horse",
}
assert expected.issubset(set(FAL_FAMILIES.keys())), (
f"missing families: {expected - set(FAL_FAMILIES.keys())}"
)
for fid, meta in FAL_FAMILIES.items():
assert meta.get("text_endpoint"), f"{fid} missing text_endpoint"
assert meta.get("image_endpoint"), f"{fid} missing image_endpoint"
assert meta["text_endpoint"] != meta["image_endpoint"]
assert meta.get("tier") in {"cheap", "premium"}, (
f"{fid} has invalid tier"
)
def test_kling_4k_uses_start_image_url():
"""Kling v3 4K's image-to-video endpoint expects start_image_url,
not image_url. The family must declare image_param_key='start_image_url'."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["kling-v3-4k"]
assert meta.get("image_param_key") == "start_image_url"
payload = _build_payload(
meta,
prompt="x",
image_url="https://example.com/i.png",
duration=5,
aspect_ratio="16:9",
resolution="720p",
negative_prompt=None,
audio=None,
seed=None,
)
assert payload.get("start_image_url") == "https://example.com/i.png"
assert "image_url" not in payload
def test_fal_list_models_advertises_both_modalities():
from plugins.video_gen.fal import FALVideoGenProvider
models = FALVideoGenProvider().list_models()
for m in models:
assert set(m["modalities"]) == {"text", "image"}, (
f"{m['id']} doesn't advertise both modalities — every family "
f"should have t2v + i2v"
)
def test_fal_unavailable_without_key(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
from plugins.video_gen import fal as fal_plugin
monkeypatch.delenv("FAL_KEY", raising=False)
# Also ensure managed gateway is unavailable
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
assert FALVideoGenProvider().is_available() is False
def test_fal_generate_requires_fal_key(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
from plugins.video_gen import fal as fal_plugin
monkeypatch.delenv("FAL_KEY", raising=False)
# Also ensure managed gateway is unavailable
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
result = FALVideoGenProvider().generate("a happy dog")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_fal_available_via_gateway(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
from plugins.video_gen import fal as fal_plugin
monkeypatch.delenv("FAL_KEY", raising=False)
monkeypatch.setattr(
fal_plugin,
"_resolve_managed_fal_video_gateway",
lambda: object(), # truthy sentinel — gateway is available
)
assert FALVideoGenProvider().is_available() is True
class TestFamilyRouting:
"""The headline behavior: image_url presence picks the endpoint."""
@pytest.fixture
def with_fake_fal(self, monkeypatch):
"""Stub fal_client.submit to capture which endpoint we hit."""
import sys
import types
captured = {"endpoint": None, "arguments": None}
class FakeHandle:
def get(self):
return {"video": {"url": "https://fake/out.mp4"}}
fake = types.ModuleType("fal_client")
def _submit(endpoint, arguments=None, headers=None):
captured["endpoint"] = endpoint
captured["arguments"] = arguments
return FakeHandle()
fake.submit = _submit # type: ignore
monkeypatch.setitem(sys.modules, "fal_client", fake)
# Reset the lazy global so it picks up our stub
from plugins.video_gen import fal as fal_plugin
fal_plugin._fal_client = None
# Also reset the managed client cache
fal_plugin._managed_fal_video_client = None
fal_plugin._managed_fal_video_client_config = None
monkeypatch.setenv("FAL_KEY", "test")
# Force direct mode — no managed gateway
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
return captured
def test_text_to_video_routes_to_text_endpoint(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"a dog running",
model="pixverse-v6",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/text-to-video"
assert result["modality"] == "text"
assert with_fake_fal["arguments"]["prompt"] == "a dog running"
assert "image_url" not in with_fake_fal["arguments"]
def test_image_to_video_routes_to_image_endpoint(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"animate this dog",
model="pixverse-v6",
image_url="https://example.com/dog.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/image-to-video"
assert result["modality"] == "image"
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
def test_default_family_text_routing(self, with_fake_fal):
"""No model arg → DEFAULT_MODEL → text-to-video endpoint."""
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate("a dog")
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_default_family_image_routing(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate(
"animate this",
image_url="https://example.com/i.png",
)
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["image_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_unknown_family_falls_back_to_default(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate(
"x",
model="not-a-real-family",
)
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_premium_seedance_routing(self, with_fake_fal):
"""Sanity check the premium-tier seedance routes correctly."""
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"a dog",
model="seedance-2.0",
image_url="https://example.com/dog.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "bytedance/seedance-2.0/image-to-video"
# Seedance uses regular image_url (not start_image_url)
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
def test_kling_4k_remaps_image_param(self, with_fake_fal):
"""Kling v3 4K image-to-video receives start_image_url, not image_url."""
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"x",
model="kling-v3-4k",
image_url="https://example.com/frame.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/kling-video/v3/4k/image-to-video"
assert with_fake_fal["arguments"].get("start_image_url") == "https://example.com/frame.png"
assert "image_url" not in with_fake_fal["arguments"]
class TestPayloadBuilder:
def test_drops_unsupported_keys(self):
"""Veo enum-clamps duration, supports aspect+resolution+audio+neg."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["veo3.1"]
p = _build_payload(
meta,
prompt="x",
image_url=None,
duration=12, # not in enum (4,6,8) — snap to 8
aspect_ratio="16:9",
resolution="720p",
negative_prompt="ugly",
audio=True,
seed=42,
)
assert p["prompt"] == "x"
assert p["duration"] == "8s" # veo3.1 uses "Ns" format per FAL API
assert p["aspect_ratio"] == "16:9"
assert p["resolution"] == "720p"
assert p["generate_audio"] is True
assert p["negative_prompt"] == "ugly"
assert p["seed"] == 42
def test_pixverse_range_clamps_correctly(self):
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["pixverse-v6"]
p = _build_payload(
meta,
prompt="x",
image_url="https://i.png",
duration=99, # over max → 15
aspect_ratio="16:9",
resolution="540p",
negative_prompt=None,
audio=None,
seed=None,
)
assert p["duration"] == "15"
def test_kling_4k_clamps_below_min(self):
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["kling-v3-4k"]
p = _build_payload(
meta,
prompt="x",
image_url="https://i.png",
duration=1, # below min (3) → 3
aspect_ratio="16:9",
resolution="720p",
negative_prompt=None,
audio=None,
seed=None,
)
assert p["duration"] == "3"
def test_ltx_omits_duration_aspect_resolution(self):
"""LTX 2.3 doesn't declare duration/aspect/resolution enums —
the payload should NOT include those keys (let FAL default)."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["ltx-2.3"]
p = _build_payload(
meta,
prompt="x",
image_url=None,
duration=8,
aspect_ratio="16:9",
resolution="720p",
negative_prompt="ugly",
audio=True,
seed=None,
)
assert "duration" not in p
assert "aspect_ratio" not in p
assert "resolution" not in p
# But audio + negative are advertised
assert p["generate_audio"] is True
assert p["negative_prompt"] == "ugly"
def test_happy_horse_minimal_payload(self):
"""Happy Horse has sparse docs — payload should be minimal."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["happy-horse"]
p = _build_payload(
meta,
prompt="a horse galloping",
image_url=None,
duration=8,
aspect_ratio="16:9",
resolution="720p",
negative_prompt="watermark",
audio=True,
seed=None,
)
# Only prompt — no payload bloat for fields we can't verify
assert p == {"prompt": "a horse galloping"}
tests/plugins/video_gen/test_xai_plugin.py
+150
View File
@@ -0,0 +1,150 @@
"""Smoke tests for the xAI video gen plugin — load & register surface."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
def test_xai_provider_registers():
from plugins.video_gen.xai import XAIVideoGenProvider
provider = XAIVideoGenProvider()
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("xai") is provider
assert provider.display_name == "xAI"
assert provider.default_model() == "grok-imagine-video"
def test_xai_provider_lists_text_and_current_image_video_models():
from plugins.video_gen.xai import XAIVideoGenProvider
models = XAIVideoGenProvider().list_models()
ids = [model["id"] for model in models]
assert ids[0] == "grok-imagine-video"
assert ids[1] == "grok-imagine-video-1.5-preview"
assert models[1]["modalities"] == ["image"]
assert models[1]["aliases"] == ["grok-imagine-video-1.5-2026-05-30"]
def test_xai_routes_default_models_by_modality():
from plugins.video_gen.xai import _resolve_model_for_modality
assert _resolve_model_for_modality(
"grok-imagine-video",
modality="text",
explicit_model=False,
) == "grok-imagine-video"
assert _resolve_model_for_modality(
"grok-imagine-video",
modality="image",
explicit_model=False,
) == "grok-imagine-video-1.5-preview"
assert _resolve_model_for_modality(
"grok-imagine-video-1.5-preview",
modality="text",
explicit_model=False,
) == "grok-imagine-video"
assert _resolve_model_for_modality(
"grok-imagine-video-1.5-preview",
modality="text",
explicit_model=True,
) == "grok-imagine-video-1.5-preview"
def test_xai_capabilities_text_and_image_only():
"""xAI was previously advertised with edit/extend operations. The
simplified surface only exposes text-to-video and image-to-video —
confirm those are the only modalities advertised."""
from plugins.video_gen.xai import XAIVideoGenProvider
caps = XAIVideoGenProvider().capabilities()
assert caps["modalities"] == ["text", "image"]
# No 'operations' key in the simplified surface
assert "operations" not in caps
assert caps["max_reference_images"] == 7
def test_xai_unavailable_without_key(monkeypatch):
from plugins.video_gen.xai import XAIVideoGenProvider
monkeypatch.delenv("XAI_API_KEY", raising=False)
assert XAIVideoGenProvider().is_available() is False
def test_xai_generate_requires_xai_key(monkeypatch):
from plugins.video_gen.xai import XAIVideoGenProvider
monkeypatch.delenv("XAI_API_KEY", raising=False)
result = XAIVideoGenProvider().generate("a happy dog")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_xai_available_with_oauth_only(monkeypatch):
"""The plugin must honour xAI Grok OAuth credentials, not just
XAI_API_KEY. Otherwise the agent's tool-availability check filters
``video_generate`` out of the toolbelt and the agent silently falls
back to whatever skill advertises video generation (e.g. comfyui).
"""
import plugins.video_gen.xai as xai_plugin
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(
"tools.xai_http.resolve_xai_http_credentials",
lambda: {
"provider": "xai-oauth",
"api_key": "oauth-bearer-token",
"base_url": "https://api.x.ai/v1",
},
)
assert xai_plugin.XAIVideoGenProvider().is_available() is True
def test_xai_resolved_credentials_threaded_through_request(monkeypatch):
"""OAuth-resolved creds must reach the HTTP layer — bug class where
``is_available()`` says yes but the request still hits with no key.
"""
import plugins.video_gen.xai as xai_plugin
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(
"tools.xai_http.resolve_xai_http_credentials",
lambda: {
"provider": "xai-oauth",
"api_key": "oauth-bearer-token",
"base_url": "https://api.x.ai/v1",
},
)
api_key, base_url = xai_plugin._resolve_xai_credentials()
assert api_key == "oauth-bearer-token"
assert base_url == "https://api.x.ai/v1"
headers = xai_plugin._xai_headers(api_key)
assert headers["Authorization"] == "Bearer oauth-bearer-token"
def test_xai_no_operation_kwarg():
"""The ABC's generate() signature no longer accepts 'operation'.
Passing it through **kwargs should be ignored (forward-compat)."""
from plugins.video_gen.xai import XAIVideoGenProvider
# We're not actually hitting the network — just verify the call
# doesn't TypeError on the unexpected kwarg.
# Will fail with auth_required (no XAI_API_KEY), but should NOT
# fail with TypeError.
result = XAIVideoGenProvider().generate("x", operation="generate")
assert result["success"] is False
# auth_required, NOT some signature error
assert result["error_type"] in {"auth_required", "api_error"}
tests/plugins/video_gen/test_xai_plugin_integration.py
+215
View File
@@ -0,0 +1,215 @@
"""Integration tests for the xAI video gen plugin's simplified surface.
xAI exposes only text-to-video and image-to-video through the unified
``video_generate`` tool. We assert the endpoint hit and the payload shape
because routing is the part most likely to break silently.
"""
from __future__ import annotations
import asyncio
import json
from typing import Any, Dict, List, Optional
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
class _FakeResponse:
def __init__(self, status: int = 200, payload: Optional[Dict[str, Any]] = None):
self.status_code = status
self._payload = payload or {}
self.text = json.dumps(self._payload)
def raise_for_status(self):
if self.status_code >= 400:
import httpx
raise httpx.HTTPStatusError("err", request=None, response=self) # type: ignore
def json(self):
return self._payload
class _FakeAsyncClient:
def __init__(self):
self.posts: List[Dict[str, Any]] = []
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return None
async def post(self, url, headers=None, json=None, timeout=None):
self.posts.append({"url": url, "json": json})
return _FakeResponse(200, {"request_id": "req-123"})
async def get(self, url, headers=None, timeout=None):
return _FakeResponse(200, {
"status": "done",
"video": {"url": "https://xai-cdn/out.mp4", "duration": 8},
"model": self.posts[-1]["json"]["model"],
})
@pytest.fixture
def xai_provider(monkeypatch):
monkeypatch.setenv("XAI_API_KEY", "test-key")
import plugins.video_gen.xai as xai_plugin
captured: Dict[str, _FakeAsyncClient] = {}
def _client_factory():
captured["client"] = _FakeAsyncClient()
return captured["client"]
monkeypatch.setattr(xai_plugin.httpx, "AsyncClient", _client_factory)
async def _no_sleep(*a, **k):
return None
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
provider = xai_plugin.XAIVideoGenProvider()
return provider, captured
def _last_post(captured) -> Dict[str, Any]:
return captured["client"].posts[-1]
class TestXAIEndpoint:
"""xAI uses one endpoint — ``/videos/generations`` — for both modes."""
def test_text_to_video_hits_generations(self, xai_provider):
provider, captured = xai_provider
result = provider.generate("a dog on a skateboard")
assert result["success"] is True
assert _last_post(captured)["url"].endswith("/videos/generations")
assert result["modality"] == "text"
def test_image_to_video_hits_generations(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"animate this",
image_url="https://example.com/cat.png",
)
assert result["success"] is True
assert _last_post(captured)["url"].endswith("/videos/generations")
assert result["modality"] == "image"
class TestXAIPayload:
def test_text_payload_has_no_image_field(self, xai_provider):
provider, captured = xai_provider
provider.generate("a dog at sunset")
payload = _last_post(captured)["json"]
assert payload["model"] == "grok-imagine-video"
assert payload["prompt"] == "a dog at sunset"
assert "image" not in payload
assert "reference_images" not in payload
def test_image_payload_has_image_field(self, xai_provider):
provider, captured = xai_provider
provider.generate("animate this", image_url="https://example.com/cat.png")
payload = _last_post(captured)["json"]
assert payload["model"] == "grok-imagine-video-1.5-preview"
assert payload["image"] == {"url": "https://example.com/cat.png"}
def test_local_image_path_is_sent_as_data_uri(self, xai_provider, tmp_path):
provider, captured = xai_provider
image_path = tmp_path / "frame.png"
image_path.write_bytes(b"\x89PNG\r\n\x1a\nfake")
provider.generate("animate this", image_url=str(image_path))
payload = _last_post(captured)["json"]
assert payload["model"] == "grok-imagine-video-1.5-preview"
assert payload["image"]["url"].startswith("data:image/png;base64,")
def test_explicit_model_override_is_honored_for_image(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"animate this",
image_url="https://example.com/cat.png",
model="grok-imagine-video",
_model_override_explicit=True,
)
payload = _last_post(captured)["json"]
assert payload["model"] == "grok-imagine-video"
def test_reference_images_payload(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"keep this character",
reference_image_urls=[
"https://example.com/a.png",
"https://example.com/b.png",
],
)
payload = _last_post(captured)["json"]
assert payload["reference_images"] == [
{"url": "https://example.com/a.png"},
{"url": "https://example.com/b.png"},
]
class TestXAIValidation:
def test_missing_prompt_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate("")
assert result["success"] is False
assert result["error_type"] == "missing_prompt"
# Never hit the network
assert "client" not in captured or not captured["client"].posts
def test_image_plus_refs_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"x",
image_url="https://example.com/i.png",
reference_image_urls=["https://example.com/r.png"],
)
assert result["success"] is False
assert result["error_type"] == "conflicting_inputs"
assert "client" not in captured or not captured["client"].posts
def test_too_many_references_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"x",
reference_image_urls=[f"https://example.com/r{i}.png" for i in range(8)],
)
assert result["success"] is False
assert result["error_type"] == "too_many_references"
class TestXAIClamping:
def test_duration_clamped_to_15(self, xai_provider):
provider, captured = xai_provider
provider.generate("x", duration=30)
assert _last_post(captured)["json"]["duration"] == 15
def test_duration_clamped_when_refs_present(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"x",
duration=15,
reference_image_urls=["https://example.com/r.png"],
)
# refs present caps to 10
assert _last_post(captured)["json"]["duration"] == 10
def test_invalid_aspect_ratio_soft_clamps(self, xai_provider):
provider, captured = xai_provider
provider.generate("x", aspect_ratio="21:9")
assert _last_post(captured)["json"]["aspect_ratio"] == "16:9"
tests/plugins/web/__init__.py
View File
tests/plugins/web/test_parallel_keyless_mcp.py
+383
View File
@@ -0,0 +1,383 @@
"""Keyless Parallel search via the free hosted Search MCP.
Covers the transport added in ``plugins/web/parallel/provider.py`` that lets
``web_search`` work with no ``PARALLEL_API_KEY``:
- ``_mcp_headers`` — Bearer attached only when a key is held
- ``_decode_mcp_envelope`` — plain-JSON and SSE (``data:``) response bodies
- ``_mcp_payload`` — structuredContent preferred, text-block JSON fallback, errors
- ``_mcp_web_search`` — full handshake (mocked transport) → standard search shape
- ``ParallelWebSearchProvider.search`` — keyless path routes to the MCP
"""
from __future__ import annotations
import asyncio
import json
from unittest.mock import patch
import pytest
import plugins.web.parallel.provider as pp
# ─── _mcp_headers ──────────────────────────────────────────────────────────
class TestMcpHeaders:
def test_anonymous_has_no_authorization(self):
h = pp._mcp_headers(session_id=None, api_key=None)
assert "Authorization" not in h
assert h["Accept"] == "application/json, text/event-stream"
assert "Mcp-Session-Id" not in h
def test_user_agent_is_generic_not_hermes(self):
# Telemetry policy: no third-party usage attribution without opt-in.
# The UA must be set (not python-httpx default) but must not name
# hermes, on both the anonymous and keyed paths.
for ua in (
pp._mcp_headers(session_id=None, api_key=None)["User-Agent"],
pp._mcp_headers(session_id="sid", api_key="pk-live")["User-Agent"],
):
assert ua == f"{pp._MCP_CLIENT_NAME}/{pp._MCP_CLIENT_VERSION}"
assert "hermes" not in ua.lower()
def test_session_id_and_bearer_when_present(self):
h = pp._mcp_headers(session_id="sid-123", api_key="pk-live")
assert h["Mcp-Session-Id"] == "sid-123"
assert h["Authorization"] == "Bearer pk-live"
# ─── SSE / JSON-RPC parsing ──────────────────────────────────────────────────
class TestMcpResponseParsing:
def test_plain_json_matched_by_id(self):
body = '{"jsonrpc":"2.0","id":"abc","result":{"ok":true}}'
assert pp._mcp_response_envelope(body, "abc")["result"]["ok"] is True
def test_sse_selects_response_for_request_id_skipping_notifications(self):
# A progress notification (no id) precedes the real result; an unrelated
# response id is also present. We must pick the one matching our id.
body = (
'event: message\ndata: {"jsonrpc":"2.0","method":"notifications/progress","params":{"p":1}}\n\n'
'event: message\ndata: {"jsonrpc":"2.0","id":"other","result":{"ok":false}}\n\n'
'event: message\ndata: {"jsonrpc":"2.0","id":"req-1","result":{"ok":true}}\n\n'
)
env = pp._mcp_response_envelope(body, "req-1")
assert env["result"]["ok"] is True
def test_sse_multiline_data_concatenated(self):
body = 'data: {"jsonrpc":"2.0","id":"x",\ndata: "result":{"n":42}}\n\n'
assert pp._mcp_response_envelope(body, "x")["result"]["n"] == 42
def test_falls_back_to_last_result_when_id_absent(self):
body = '{"jsonrpc":"2.0","id":"server-chose","result":{"ok":true}}'
# request id doesn't match, but there's a single result → use it
assert pp._mcp_response_envelope(body, "mismatch")["result"]["ok"] is True
def test_empty_body(self):
assert pp._mcp_response_envelope("", "x") == {}
assert pp._mcp_response_envelope(" ", "x") == {}
def test_batched_json_array_flattened(self):
# Streamable HTTP may batch messages into a JSON array.
body = ('[{"jsonrpc":"2.0","method":"notifications/progress"},'
'{"jsonrpc":"2.0","id":"req-9","result":{"ok":true}}]')
assert pp._mcp_response_envelope(body, "req-9")["result"]["ok"] is True
def test_batched_sse_data_array_flattened(self):
body = 'data: [{"jsonrpc":"2.0","id":"a","result":{"n":1}}]\n\n'
assert pp._mcp_response_envelope(body, "a")["result"]["n"] == 1
# ─── _mcp_payload ────────────────────────────────────────────────────────────
class TestMcpPayload:
def test_prefers_structured_content(self):
env = {"result": {"structuredContent": {"results": [{"url": "u"}]},
"content": [{"type": "text", "text": "ignored"}]}}
assert pp._mcp_payload(env) == {"results": [{"url": "u"}]}
def test_parses_text_block_json(self):
inner = {"search_id": "s1", "results": [{"url": "u", "title": "t"}]}
env = {"result": {"content": [{"type": "text", "text": json.dumps(inner)}]}}
assert pp._mcp_payload(env)["search_id"] == "s1"
def test_raises_on_jsonrpc_error(self):
with pytest.raises(RuntimeError, match="Parallel MCP error"):
pp._mcp_payload({"error": {"code": -32000, "message": "boom"}})
def test_raises_on_tool_iserror(self):
with pytest.raises(RuntimeError, match="Parallel MCP tool error"):
pp._mcp_payload({"result": {"isError": True, "content": []}})
# ─── _mcp_web_search (mocked transport) ──────────────────────────────────────
class _FakeResponse:
def __init__(self, *, text="", headers=None):
self.text = text
self.headers = headers or {}
def raise_for_status(self):
return None
class _FakeClient:
"""Stands in for httpx.Client: replays init → ack → tools/call."""
def __init__(self, search_payload, init_session_id="server-sid"):
self._search_payload = search_payload
self._init_session_id = init_session_id
self.calls = []
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def post(self, url, headers=None, json=None):
self.calls.append({"headers": headers, "json": json})
req = json or {}
method = req.get("method")
req_id = req.get("id")
if method == "initialize":
# Echo the request id, as the real server does.
return _FakeResponse(
text=json_dumps({"jsonrpc": "2.0", "id": req_id,
"result": {"protocolVersion": "2099-01-01"}}),
headers=(
{"mcp-session-id": self._init_session_id}
if self._init_session_id is not None
else {}
),
)
if method == "notifications/initialized":
return _FakeResponse(text="")
# tools/call
envelope = {"jsonrpc": "2.0", "id": req_id, "result": {
"content": [{"type": "text", "text": json_dumps(self._search_payload)}],
}}
return _FakeResponse(text=json_dumps(envelope))
def json_dumps(obj):
return json.dumps(obj)
class TestMcpWebSearch:
def _payload(self, n):
return {"search_id": "s", "results": [
{"url": f"https://ex/{i}", "title": f"t{i}",
"excerpts": [f"a{i}", f"b{i}"]}
for i in range(n)
]}
def test_returns_standard_shape_and_handshake(self):
fake = _FakeClient(self._payload(3))
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_search("hello", limit=5, api_key=None)
assert out["success"] is True
# Free-tier results credit Parallel.
assert "Parallel" in out["attribution"]
web = out["data"]["web"]
assert [r["position"] for r in web] == [1, 2, 3]
assert web[0]["url"] == "https://ex/0"
assert web[0]["description"] == "a0 b0" # excerpts joined
# handshake order
methods = [c["json"].get("method") for c in fake.calls]
assert methods == ["initialize", "notifications/initialized", "tools/call"]
# session id from the initialize response header is reused
assert fake.calls[-1]["headers"]["Mcp-Session-Id"] == "server-sid"
def test_stateless_server_no_session_header_not_invented(self):
# A stateless Streamable-HTTP server may omit mcp-session-id on
# initialize; we must NOT invent one (sending an unissued session id can
# get follow-up requests rejected). The follow-ups carry no header.
fake = _FakeClient(self._payload(1), init_session_id=None)
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_search("hello", limit=5, api_key=None)
assert out["success"] is True
follow_ups = [c for c in fake.calls if c["json"].get("method") != "initialize"]
assert follow_ups, "expected notifications/initialized + tools/call"
assert all("Mcp-Session-Id" not in c["headers"] for c in follow_ups)
# anonymous → no Authorization on any call
assert all("Authorization" not in c["headers"] for c in fake.calls)
# tools/call mirrors query into objective + search_queries
args = fake.calls[-1]["json"]["params"]["arguments"]
assert args["objective"] == "hello"
assert args["search_queries"] == ["hello"]
def test_limit_is_applied_client_side(self):
fake = _FakeClient(self._payload(10))
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_search("q", limit=2, api_key=None)
assert len(out["data"]["web"]) == 2
def test_bearer_attached_when_key_present(self):
fake = _FakeClient(self._payload(1))
with patch.object(pp.httpx, "Client", return_value=fake):
pp._mcp_web_search("q", limit=1, api_key="pk-live")
assert all(c["headers"]["Authorization"] == "Bearer pk-live" for c in fake.calls)
def test_negotiated_protocol_version_echoed_post_init(self):
fake = _FakeClient(self._payload(1))
with patch.object(pp.httpx, "Client", return_value=fake):
pp._mcp_web_search("q", limit=1, api_key=None)
# initialize request doesn't carry the (not-yet-negotiated) version...
assert "MCP-Protocol-Version" not in fake.calls[0]["headers"]
# ...but notifications/initialized and tools/call echo the negotiated one.
assert fake.calls[1]["headers"]["MCP-Protocol-Version"] == "2099-01-01"
assert fake.calls[-1]["headers"]["MCP-Protocol-Version"] == "2099-01-01"
# ─── provider.search keyless routing ─────────────────────────────────────────
class TestProviderKeylessSearch:
def test_search_without_key_uses_mcp(self, monkeypatch):
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
captured = {}
def _fake(query, limit, api_key):
captured.update(query=query, limit=limit, api_key=api_key)
return {"success": True, "data": {"web": []}}
monkeypatch.setattr(pp, "_mcp_web_search", _fake)
out = pp.ParallelWebSearchProvider().search("kittens", limit=4)
assert out["success"] is True
assert captured == {"query": "kittens", "limit": 4, "api_key": None}
def test_is_available_reflects_key(self, monkeypatch):
# is_available() gates the registry's active-provider walk + picker, so
# it's key-based (keyless dispatch is handled by _get_backend, not this).
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
assert pp.ParallelWebSearchProvider().is_available() is False
monkeypatch.setenv("PARALLEL_API_KEY", "k")
assert pp.ParallelWebSearchProvider().is_available() is True
# ─── web_fetch (keyless extract) ─────────────────────────────────────────────
class TestMcpWebFetch:
def _payload(self, urls):
return {"extract_id": "e1", "results": [
{"url": u, "title": f"T{i}", "publish_date": None,
"excerpts": [f"chunk-a-{i}", f"chunk-b-{i}"]}
for i, u in enumerate(urls)
]}
def test_maps_to_extract_shape(self):
urls = ["https://a.test", "https://b.test"]
fake = _FakeClient(self._payload(urls))
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_fetch(urls, api_key=None)
assert [r["url"] for r in out] == urls
assert out[0]["content"] == "chunk-a-0\n\nchunk-b-0"
assert out[0]["raw_content"] == out[0]["content"]
assert out[0]["metadata"] == {"sourceURL": "https://a.test", "title": "T0"}
# tools/call targeted web_fetch, requesting full page bodies.
args = fake.calls[-1]["json"]["params"]
assert args["name"] == "web_fetch"
assert args["arguments"]["urls"] == urls
assert args["arguments"]["full_content"] is True
assert args["arguments"]["session_id"].startswith(f"{pp._MCP_CLIENT_NAME}-")
def test_prefers_full_content_over_excerpts(self):
payload = {"results": [
{"url": "https://a.test", "title": "T",
"excerpts": ["snippet"], "full_content": "the entire page body"},
]}
fake = _FakeClient(payload)
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_fetch(["https://a.test"], api_key=None)
assert out[0]["content"] == "the entire page body"
def test_missing_url_becomes_error_entry(self):
# Server returns only one of the two requested URLs.
fake = _FakeClient(self._payload(["https://a.test"]))
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_fetch(["https://a.test", "https://missing.test"], api_key=None)
assert len(out) == 2
missing = [r for r in out if r["url"] == "https://missing.test"][0]
assert "error" in missing
assert missing["content"] == ""
def test_preserves_order_and_duplicate_inputs(self):
# MCP returns each unique URL once; output must still be one row per
# input, in order, including the duplicate.
fake = _FakeClient(self._payload(["https://a.test", "https://b.test"]))
urls = ["https://b.test", "https://a.test", "https://b.test"]
with patch.object(pp.httpx, "Client", return_value=fake):
out = pp._mcp_web_fetch(urls, api_key=None)
assert [r["url"] for r in out] == urls # one row per input, in order
assert all("error" not in r for r in out) # all three resolved
def test_extract_without_key_uses_web_fetch(self, monkeypatch):
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
captured = {}
def _fake(urls, api_key):
captured.update(urls=list(urls), api_key=api_key)
return [{"url": urls[0], "title": "", "content": "x",
"raw_content": "x", "metadata": {}}]
monkeypatch.setattr(pp, "_mcp_web_fetch", _fake)
out = asyncio.run(pp.ParallelWebSearchProvider().extract(["https://x.test"]))
assert out[0]["content"] == "x"
assert captured == {"urls": ["https://x.test"], "api_key": None}
# ─── keyed v1 REST search ────────────────────────────────────────────────────
class TestKeyedV1Search:
def test_passes_max_results_and_omits_branding(self, monkeypatch):
monkeypatch.setenv("PARALLEL_API_KEY", "pk-live")
monkeypatch.delenv("PARALLEL_SEARCH_MODE", raising=False)
captured = {}
class _Res:
def __init__(self, url):
self.url, self.title, self.excerpts = url, "T", ["x"]
class _Resp:
results = [_Res(f"https://r/{i}") for i in range(10)]
class _Client:
def search(self, **kw):
captured.update(kw)
return _Resp()
monkeypatch.setattr(pp, "_get_sync_client", lambda: _Client())
out = pp.ParallelWebSearchProvider().search("q", limit=7)
assert out["success"] is True
# honors the caller's limit via advanced_settings.max_results
assert captured["advanced_settings"] == {"max_results": 7}
assert captured["mode"] == "advanced" # v1 default
assert captured["session_id"].startswith(f"{pp._MCP_CLIENT_NAME}-") # per-call id
assert len(out["data"]["web"]) == 7 # client-side slice
# paid path: no free-tier attribution, no [Parallel] label signal
assert "attribution" not in out
assert "provider" not in out
# ─── v1 search mode mapping ──────────────────────────────────────────────────
class TestResolveSearchMode:
@pytest.mark.parametrize("env,expected", [
(None, "advanced"), # default
("advanced", "advanced"),
("basic", "basic"),
("fast", "basic"), # legacy → basic
("one-shot", "basic"), # legacy → basic
("agentic", "advanced"), # legacy → advanced
("garbage", "advanced"), # invalid → default
("BASIC", "basic"), # case-insensitive
])
def test_mode_mapping(self, monkeypatch, env, expected):
if env is None:
monkeypatch.delenv("PARALLEL_SEARCH_MODE", raising=False)
else:
monkeypatch.setenv("PARALLEL_SEARCH_MODE", env)
assert pp._resolve_search_mode() == expected
tests/plugins/web/test_web_search_provider_plugins.py
+519
View File
@@ -0,0 +1,519 @@
"""Plugin-side tests for the web search provider migration (PR #25182).
Covers:
- All eight bundled plugins (brave-free, ddgs, searxng, exa, parallel,
tavily, firecrawl, xai) instantiate and self-report the expected
capabilities + ABC-derived defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The web_search_registry resolves an active provider in the documented
scenarios (explicit config wins ignoring availability, fallback walks
legacy preference filtered by availability, unknown name falls back).
- Plugin response shapes match the legacy bit-for-bit contract.
Per the dev skill: these tests use *real* imports from the plugin
modules — no mocking of provider classes themselves — so the test
catches drift in the ABC interface, the registry, and the plugin
glue layer simultaneously.
"""
from __future__ import annotations
import asyncio
import inspect
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every web-provider env var so is_available() returns False."""
for k in (
"BRAVE_SEARCH_API_KEY",
"SEARXNG_URL",
"TAVILY_API_KEY",
"TAVILY_BASE_URL",
"EXA_API_KEY",
"PARALLEL_API_KEY",
"PARALLEL_SEARCH_MODE",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_USER_TOKEN",
"XAI_API_KEY",
):
monkeypatch.delenv(k, raising=False)
def _ensure_plugins_loaded() -> None:
"""Idempotently load plugins so the registry is populated."""
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
# ---------------------------------------------------------------------------
# Per-plugin discovery + capability flags
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each test starts with a clean web-provider env."""
_clear_web_env(monkeypatch)
class TestBundledPluginsRegister:
"""All eight bundled web plugins discover and register correctly."""
def test_all_seven_plugins_present_in_registry(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import list_providers
names = sorted(p.name for p in list_providers())
assert names == [
"brave-free",
"ddgs",
"exa",
"firecrawl",
"parallel",
"searxng",
"tavily",
"xai",
]
@pytest.mark.parametrize(
"plugin_name,expected_search,expected_extract",
[
("brave-free", True, False),
("ddgs", True, False),
("searxng", True, False),
("exa", True, True),
("parallel", True, True),
("tavily", True, True),
("firecrawl", True, True),
# xai: search-only via Grok's agentic web_search tool.
("xai", True, False),
],
)
def test_capability_flags_match_spec(
self,
plugin_name: str,
expected_search: bool,
expected_extract: bool,
) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None, f"plugin {plugin_name!r} not registered"
assert provider.supports_search() is expected_search
assert provider.supports_extract() is expected_extract
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"],
)
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
assert provider.name == plugin_name
assert provider.display_name # any non-empty string
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"],
)
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
"""``get_setup_schema()`` returns a dict the picker can consume."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
schema = provider.get_setup_schema()
assert isinstance(schema, dict)
assert "name" in schema
assert "env_vars" in schema
# ---------------------------------------------------------------------------
# is_available() behavior
# ---------------------------------------------------------------------------
class TestIsAvailable:
"""Each plugin's ``is_available()`` returns False without env config."""
def test_brave_free_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("brave-free")
assert p is not None
assert p.is_available() is False # no BRAVE_SEARCH_API_KEY
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "real")
assert p.is_available() is True
def test_searxng_requires_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("searxng")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080")
assert p.is_available() is True
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("TAVILY_API_KEY", "real")
assert p.is_available() is True
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("EXA_API_KEY", "real")
assert p.is_available() is True
def test_parallel_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""is_available() is key-based — it gates the registry's active-provider
walk/picker. (Keyless search/extract still work via the free MCP through
_get_backend's terminal default, independent of this flag.)
"""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
assert p.is_available() is False
monkeypatch.setenv("PARALLEL_API_KEY", "real")
assert p.is_available() is True
def test_firecrawl_requires_either_key_or_url(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert p.is_available() is False
# Either FIRECRAWL_API_KEY or FIRECRAWL_API_URL lights it up.
monkeypatch.setenv("FIRECRAWL_API_KEY", "real")
assert p.is_available() is True
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
monkeypatch.setenv("FIRECRAWL_API_URL", "http://localhost:3002")
assert p.is_available() is True
def test_ddgs_always_available_when_package_importable(self) -> None:
"""DDGS is the always-on fallback — no API key required.
It may report unavailable if the ``ddgs`` package itself isn't
installed in the env (legitimate — the plugin's post_setup hook
triggers pip install on first selection). We only assert that
is_available() doesn't raise.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("ddgs")
assert p is not None
# Truthy or falsy, just must not raise.
_ = bool(p.is_available())
def test_xai_requires_api_key_or_oauth(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""xAI needs XAI_API_KEY or OAuth tokens in auth.json."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("xai")
assert p is not None
assert p.is_available() is False # no XAI_API_KEY, no auth.json
monkeypatch.setenv("XAI_API_KEY", "real")
assert p.is_available() is True
# ---------------------------------------------------------------------------
# Registry resolution semantics (Option B — conservative smart fallback)
# ---------------------------------------------------------------------------
class TestRegistryResolution:
"""``_resolve()`` follows explicit-config + availability-filtered fallback."""
def test_explicit_configured_provider_returned_even_when_unavailable(
self,
) -> None:
"""Explicit ``web.search_backend`` wins regardless of is_available().
Without availability filtering on the explicit path, the dispatcher
would silently switch backends; with this check the dispatcher
surfaces a precise "FOO_API_KEY is not set" error instead.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
# No BRAVE_SEARCH_API_KEY (fixture cleared it).
result = _resolve("brave-free", capability="search")
assert result is not None
assert result.name == "brave-free"
# Confirm it's the unavailable one — dispatcher will surface
# a typed credential-missing error to the caller.
assert result.is_available() is False
def test_unknown_configured_name_falls_back_to_available_provider(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Typo / uninstalled plugin → walk legacy preference, pick available."""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
monkeypatch.setenv("EXA_API_KEY", "real")
result = _resolve("not-a-real-provider", capability="search")
# Either ddgs (no-key fallback) or exa (the only available
# premium provider) — both are valid. The point is the unknown
# name shouldn't return None when SOMETHING is available.
assert result is not None
assert result.is_available() is True
def test_explicit_search_only_provider_for_extract_falls_back(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Asking for extract via a search-only backend → fall back.
``brave-free`` is search-only (``supports_extract() is False``).
When the registry resolves it for an extract capability, the
explicit-config branch rejects it as capability-incompatible
and the fallback walk picks an extract-capable provider.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
monkeypatch.setenv("EXA_API_KEY", "real")
result = _resolve("brave-free", capability="extract")
# Should land on exa (only extract-capable available provider).
assert result is not None
assert result.supports_extract() is True
assert result.is_available() is True
def test_no_config_no_credentials_returns_none(
self,
) -> None:
"""No backend configured AND no available providers → typically None.
``ddgs`` is the no-credential fallback; if its ``ddgs`` Python
package is installed in the test env, ddgs will be picked.
Otherwise the resolver returns None. Either outcome is correct.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
result = _resolve(None, capability="search")
if result is not None:
# The only no-credential provider is ddgs; anything else
# means an env var leaked in.
assert result.is_available() is True
# ---------------------------------------------------------------------------
# Sync-vs-async extract detection
# ---------------------------------------------------------------------------
class TestAsyncExtractDispatch:
"""The dispatcher detects async vs sync extract methods correctly."""
def test_parallel_extract_is_async(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is True
def test_firecrawl_extract_is_async(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is True
def test_exa_extract_is_sync(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is False
def test_tavily_extract_is_sync(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is False
# ---------------------------------------------------------------------------
# Error response shape (preserved bit-for-bit from legacy)
# ---------------------------------------------------------------------------
class TestErrorResponseShapes:
"""When credentials are missing, plugins return typed errors, not raises."""
def test_brave_free_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("brave-free")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_searxng_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("searxng")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_exa_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_tavily_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_parallel_extract_keyless_uses_mcp_web_fetch(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Without a key, extract routes to the free MCP web_fetch tool rather
than erroring. The MCP transport is mocked so the test stays offline."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
import plugins.web.parallel.provider as parallel_provider
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
captured = {}
def _fake_fetch(urls, api_key):
captured["urls"] = list(urls)
captured["api_key"] = api_key
return [{"url": urls[0], "title": "Example", "content": "body",
"raw_content": "body", "metadata": {"sourceURL": urls[0]}}]
monkeypatch.setattr(parallel_provider, "_mcp_web_fetch", _fake_fetch)
p = get_provider("parallel")
assert p is not None
result = asyncio.run(p.extract(["https://example.com"]))
assert isinstance(result, list)
assert result[0]["url"] == "https://example.com"
assert result[0]["content"] == "body"
assert captured == {"urls": ["https://example.com"], "api_key": None}
def test_firecrawl_extract_returns_per_url_errors_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
# firecrawl extract returns [] when the website-policy gate rejects
# the URL, or a per-URL error dict when the gate passes but the
# firecrawl client fails. Use a URL the policy allows to make sure
# we hit the credential-missing path.
result = asyncio.run(p.extract(["https://example.com"]))
assert isinstance(result, list)
if result: # if anything came back, it should be an error entry
assert "error" in result[0]
def test_firecrawl_config_error_points_paid_users_to_nous_subscription(self, monkeypatch):
from plugins.web.firecrawl import provider as firecrawl_provider
monkeypatch.setattr(
"tools.web_tools.managed_nous_tools_enabled",
lambda: True,
raising=False,
)
with pytest.raises(ValueError) as exc_info:
firecrawl_provider._raise_web_backend_configuration_error()
message = str(exc_info.value)
assert "With your Nous subscription you can also use the Tool Gateway" in message
assert "select Nous Subscription as the web provider" in message
assert "managed Firecrawl web tools is unavailable" not in message
def test_firecrawl_config_error_uses_entitlement_message_when_not_paid(self, monkeypatch):
from plugins.web.firecrawl import provider as firecrawl_provider
monkeypatch.setattr(
"tools.web_tools.managed_nous_tools_enabled",
lambda: False,
raising=False,
)
monkeypatch.setattr(
"tools.web_tools.nous_tool_gateway_unavailable_message",
lambda capability: f"{capability} denied by test entitlement.",
raising=False,
)
with pytest.raises(ValueError) as exc_info:
firecrawl_provider._raise_web_backend_configuration_error()
assert "managed Firecrawl web tools denied by test entitlement" in str(exc_info.value)
def test_xai_search_returns_error_dict_when_unconfigured(self) -> None:
"""xAI returns a typed error dict (no XAI_API_KEY)."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("xai")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result