Sami/hermes-agent
1
0
Fork 0
forked from Zakaria/hermes-agent
Code Pull Requests Activity

Hermes-agent

Browse Source
This commit is contained in:
Zakaria
2026-06-14 14:30:48 -04:00
commit dac4b88b94
5058 changed files with 1884848 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/plugins/browser/__init__.py
View File
tests/plugins/browser/check_parity_vs_main.py
+273
View File
@@ -0,0 +1,273 @@
"""Behavior-parity check for the browser-provider plugin migration (#25214).
Spawns one subprocess per (version, scenario) cell — pinned to either
origin/main (legacy in-tree providers + class-instantiation lookup) or
this PR's worktree (plugin-based registry) via `sys.path[0]`. Each
subprocess clears all browser-related env vars + writes a config.yaml,
loads `tools.browser_tool._get_cloud_provider()`, and emits a reduced
"shape tuple" {is_local, provider_name, is_available} as JSON.
The parent process diffs the shapes per scenario. A diff means the
migration introduced an observable behaviour change vs origin/main —
which would be a real regression for users on the existing config keys.
Run from the PR worktree:
cd ~/.hermes/hermes-agent/.worktrees/browser-providers-plugin
python tests/plugins/browser/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
# Pin one path to current main, one to the PR worktree.
# ``REPO_ROOT`` is ``.../.worktrees/browser-providers-plugin``; the main
# checkout lives two levels up at ``~/.hermes/hermes-agent``.
MAIN_DIR = REPO_ROOT.parent.parent # ~/.hermes/hermes-agent
PR_DIR = REPO_ROOT # the worktree we're in
assert (MAIN_DIR / "tools" / "browser_tool.py").exists(), (
f"MAIN_DIR={MAIN_DIR} doesn't look like a hermes-agent checkout"
)
assert (PR_DIR / "tools" / "browser_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
# Reduced shape comparison — exact instance addresses obviously differ
# between subprocesses, so we compare the parts that matter for users.
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME for the config write.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear every browser-related env var so is_available() is deterministic.
for k in (
"BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", "BROWSERBASE_BASE_URL",
"BROWSER_USE_API_KEY", "BROWSER_USE_GATEWAY_URL",
"FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_BROWSER_TTL",
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
):
os.environ.pop(k, None)
# Apply per-scenario env (passed as JSON via argv[2]).
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
# Apply per-scenario config (passed as YAML body via argv[3]).
config_yaml = sys.argv[3]
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have any browser modules cached.
for name in list(sys.modules):
if name.startswith("tools.") or name.startswith("agent.") or name.startswith("plugins."):
sys.modules.pop(name, None)
from tools.browser_tool import _get_cloud_provider, _is_local_mode
provider = _get_cloud_provider()
# Pull the human-readable backend name via the API that exists on BOTH
# legacy (origin/main: CloudBrowserProvider.provider_name()) and the new
# ABC (BrowserProvider exposes provider_name() as a backward-compat alias
# returning display_name). Both shapes resolve to the same string —
# 'Browserbase' / 'Browser Use' / 'Firecrawl' — so we can compare safely.
provider_name = None
is_available = None
if provider is not None:
pn = getattr(provider, "provider_name", None)
if callable(pn):
provider_name = pn()
elif isinstance(pn, str):
provider_name = pn
is_conf = getattr(provider, "is_configured", None)
if callable(is_conf):
is_available = bool(is_conf())
shape = {
"is_local": _is_local_mode(),
"provider_name": provider_name,
"is_available": is_available,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
# (label, config.yaml body, extra env vars)
("no-config-no-env", "", {}),
("explicit-local-no-env", "browser:\n cloud_provider: local\n", {}),
(
"explicit-browserbase-no-creds",
"browser:\n cloud_provider: browserbase\n",
{},
),
(
"explicit-browserbase-with-creds",
"browser:\n cloud_provider: browserbase\n",
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
),
(
"explicit-browser-use-no-creds",
"browser:\n cloud_provider: browser-use\n",
{},
),
(
"explicit-browser-use-with-creds",
"browser:\n cloud_provider: browser-use\n",
{"BROWSER_USE_API_KEY": "k"},
),
(
"explicit-firecrawl-no-creds",
"browser:\n cloud_provider: firecrawl\n",
{},
),
(
"explicit-firecrawl-with-creds",
"browser:\n cloud_provider: firecrawl\n",
{"FIRECRAWL_API_KEY": "k"},
),
(
"no-config-bu-creds",
"",
{"BROWSER_USE_API_KEY": "k"},
),
(
"no-config-bb-creds",
"",
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
),
(
"no-config-both-creds",
"",
{
"BROWSER_USE_API_KEY": "k",
"BROWSERBASE_API_KEY": "x",
"BROWSERBASE_PROJECT_ID": "y",
},
),
(
"no-config-firecrawl-only",
"",
{"FIRECRAWL_API_KEY": "k"},
),
(
"no-config-firecrawl-and-bb",
"",
{
"FIRECRAWL_API_KEY": "k",
"BROWSERBASE_API_KEY": "x",
"BROWSERBASE_PROJECT_ID": "y",
},
),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
"""Run one (version, scenario) cell. Returns the shape dict."""
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
# Worktrees share the main repo's venv.
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
],
capture_output=True,
text=True,
timeout=30,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout,
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce_for_comparison(shape: dict) -> dict:
"""Reduce a shape dict to the parts that matter for user-visible parity.
We compare ``(is_local, provider_name, is_available)`` — the trio that
decides what the dispatcher does with each tool call. ``provider_name``
is the legacy ``provider_name()`` return value ('Browserbase' / 'Browser
Use' / 'Firecrawl'), which is identical between legacy and plugin
classes (the plugin's ``display_name`` matches the legacy
``provider_name()`` return).
"""
return {
"is_local": shape.get("is_local"),
"provider_name": shape.get("provider_name"),
"is_available": shape.get("is_available"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
failures: list[str] = []
errors: list[str] = []
for label, config_yaml, env in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce_for_comparison(main_shape)
pr_reduced = _reduce_for_comparison(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
tests/plugins/browser/test_browser_provider_plugins.py
+379
View File
@@ -0,0 +1,379 @@
"""Plugin-side tests for the browser provider migration (PR #25214).
Covers:
- All three bundled plugins (browserbase, browser-use, firecrawl)
instantiate and self-report the expected ABC defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The browser_registry resolves an active provider in the documented
scenarios:
* explicit config wins ignoring availability (so dispatcher surfaces
a typed credentials error)
* legacy preference walk: browser-use → browserbase (filtered by
availability)
* firecrawl is NOT in the legacy walk — explicit-only
* unknown name falls through to auto-detect
* ``local`` short-circuits to None
These tests use *real* imports from the plugin modules — no mocking of
provider classes themselves — so the test catches drift in the ABC
interface, the registry, and the plugin glue layer simultaneously.
Mirrors ``tests/plugins/web/test_web_search_provider_plugins.py`` from
PR #25182.
"""
from __future__ import annotations
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _clear_browser_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every browser-provider env var so is_available() returns False."""
for k in (
"BROWSERBASE_API_KEY",
"BROWSERBASE_PROJECT_ID",
"BROWSERBASE_BASE_URL",
"BROWSER_USE_API_KEY",
"BROWSER_USE_GATEWAY_URL",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_BROWSER_TTL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_USER_TOKEN",
):
monkeypatch.delenv(k, raising=False)
def _ensure_plugins_loaded() -> None:
"""Idempotently load plugins so the registry is populated."""
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
# ---------------------------------------------------------------------------
# Per-test isolation
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each test starts with a clean browser-provider env."""
_clear_browser_env(monkeypatch)
# ---------------------------------------------------------------------------
# Bundled plugins register
# ---------------------------------------------------------------------------
class TestBundledPluginsRegister:
"""All three bundled browser plugins discover and register correctly."""
def test_all_three_plugins_present_in_registry(self) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import list_providers
names = sorted(p.name for p in list_providers())
assert names == ["browser-use", "browserbase", "firecrawl"]
@pytest.mark.parametrize(
"plugin_name,expected_display",
[
("browserbase", "Browserbase"),
("browser-use", "Browser Use"),
("firecrawl", "Firecrawl"),
],
)
def test_each_plugin_has_name_and_display_name(
self, plugin_name: str, expected_display: str
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None, f"plugin {plugin_name!r} not registered"
assert provider.name == plugin_name
assert provider.display_name == expected_display
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
"""``get_setup_schema()`` returns a dict the picker can consume."""
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
schema = provider.get_setup_schema()
assert isinstance(schema, dict)
assert "name" in schema
assert "env_vars" in schema
# Every cloud-browser plugin needs the agent-browser post-setup hook
# so the picker auto-installs the CLI on selection.
assert schema.get("post_setup") == "agent_browser"
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_each_plugin_implements_full_lifecycle(self, plugin_name: str) -> None:
"""The ABC's three lifecycle methods are all overridden."""
_ensure_plugins_loaded()
from agent.browser_provider import BrowserProvider
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
# Each method must be a real override, not the ABC's NotImplementedError
# default — we check by comparing the function reference.
assert type(provider).create_session is not BrowserProvider.create_session
assert type(provider).close_session is not BrowserProvider.close_session
assert (
type(provider).emergency_cleanup is not BrowserProvider.emergency_cleanup
)
# ---------------------------------------------------------------------------
# is_available() behavior
# ---------------------------------------------------------------------------
class TestIsAvailable:
"""Each plugin's ``is_available()`` reflects env-var presence accurately."""
def test_browserbase_requires_both_api_key_and_project_id(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browserbase")
assert p is not None
assert p.is_available() is False
# API key alone is insufficient.
monkeypatch.setenv("BROWSERBASE_API_KEY", "key")
assert p.is_available() is False
# Both env vars set → available.
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
assert p.is_available() is True
def test_browserbase_project_id_alone_insufficient(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browserbase")
assert p is not None
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
assert p.is_available() is False
def test_browser_use_satisfied_by_api_key(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browser-use")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("BROWSER_USE_API_KEY", "key")
assert p.is_available() is True
def test_firecrawl_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("FIRECRAWL_API_KEY", "key")
assert p.is_available() is True
# ---------------------------------------------------------------------------
# Registry resolution semantics
# ---------------------------------------------------------------------------
class TestRegistryResolution:
"""``_resolve()`` implements the documented three-rule precedence."""
def test_resolve_none_with_no_creds_returns_none(self) -> None:
"""No config, no env → local mode (None)."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
assert _resolve(None) is None
def test_explicit_local_returns_none(self) -> None:
"""``cloud_provider: local`` is a positive choice; short-circuits to None."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
assert _resolve("local") is None
def test_explicit_browserbase_returns_provider_even_when_unavailable(self) -> None:
"""Rule 1: explicit-config wins even when credentials are missing.
This is critical — the dispatcher needs to surface a typed
credentials error rather than silently switching backends.
"""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
provider = _resolve("browserbase")
assert provider is not None
assert provider.name == "browserbase"
assert provider.is_available() is False # confirms "ignoring availability"
def test_explicit_firecrawl_returns_provider_even_when_unavailable(self) -> None:
"""Firecrawl behaves the same as browserbase under explicit config."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
provider = _resolve("firecrawl")
assert provider is not None
assert provider.name == "firecrawl"
def test_explicit_unknown_falls_back_to_auto_detect(self) -> None:
"""Rule 1 miss: unknown name → fall through to legacy walk."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
# With no credentials anywhere, auto-detect should also fail.
assert _resolve("not-a-real-provider") is None
def test_legacy_walk_prefers_browser_use_over_browserbase(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rule 3: walk order is browser-use → browserbase."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
# Both available — browser-use should win.
monkeypatch.setenv("BROWSER_USE_API_KEY", "k1")
monkeypatch.setenv("BROWSERBASE_API_KEY", "k2")
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
provider = _resolve(None)
assert provider is not None
assert provider.name == "browser-use"
def test_legacy_walk_falls_through_to_browserbase(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rule 3: browser-use unavailable → browserbase picked."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
monkeypatch.setenv("BROWSERBASE_API_KEY", "k")
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
provider = _resolve(None)
assert provider is not None
assert provider.name == "browserbase"
def test_firecrawl_not_in_legacy_walk_even_when_only_one_available(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Regression: firecrawl is NEVER auto-selected even when single-eligible.
Pre-PR-#25214, the dispatcher only auto-detected between Browser Use
and Browserbase; firecrawl was reachable solely via explicit
config. We preserve that gate because FIRECRAWL_API_KEY is shared
with the *web* firecrawl plugin — auto-routing a web-extract user
to a paid cloud browser would be a real behaviour regression.
"""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
monkeypatch.setenv("FIRECRAWL_API_KEY", "k")
# Only firecrawl is_available() — but it's not in the legacy walk.
assert _resolve(None) is None
# ---------------------------------------------------------------------------
# Legacy ABC backward-compat aliases (is_configured / provider_name)
# ---------------------------------------------------------------------------
class TestLegacyAbcAliases:
"""is_configured() and provider_name() delegate to the new API."""
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_is_configured_delegates_to_is_available(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider(plugin_name)
assert p is not None
assert p.is_configured() is p.is_available()
@pytest.mark.parametrize(
"plugin_name,expected_label",
[
("browserbase", "Browserbase"),
("browser-use", "Browser Use"),
("firecrawl", "Firecrawl"),
],
)
def test_provider_name_returns_display_name(
self, plugin_name: str, expected_label: str
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider(plugin_name)
assert p is not None
assert p.provider_name() == expected_label
# ---------------------------------------------------------------------------
# Picker integration
# ---------------------------------------------------------------------------
class TestPickerIntegration:
"""`_plugin_browser_providers()` exposes all three plugins as picker rows."""
def test_picker_rows_match_registered_plugins(self) -> None:
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
rows = _plugin_browser_providers()
names = sorted(r.get("browser_provider") for r in rows)
assert names == ["browser-use", "browserbase", "firecrawl"]
def test_picker_rows_carry_post_setup_hook(self) -> None:
"""Every browser plugin row has post_setup='agent_browser' so
selecting it triggers the agent-browser CLI install."""
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
for row in _plugin_browser_providers():
assert row.get("post_setup") == "agent_browser", (
f"plugin row {row['browser_provider']!r} missing post_setup hook"
)
def test_picker_rows_carry_browser_plugin_name_marker(self) -> None:
"""`browser_plugin_name` matches `browser_provider` so downstream
code can route through the registry when it wants to."""
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
for row in _plugin_browser_providers():
assert row.get("browser_plugin_name") == row.get("browser_provider")