forked from Zakaria/hermes-agent
Hermes-agent
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
"""Tests for the BedrockTransport."""
|
||||
|
||||
import pytest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.transports import get_transport
|
||||
from agent.transports.types import NormalizedResponse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport():
|
||||
import agent.transports.bedrock # noqa: F401
|
||||
return get_transport("bedrock_converse")
|
||||
|
||||
|
||||
class TestBedrockBasic:
|
||||
|
||||
def test_api_mode(self, transport):
|
||||
assert transport.api_mode == "bedrock_converse"
|
||||
|
||||
def test_registered(self, transport):
|
||||
assert transport is not None
|
||||
|
||||
|
||||
class TestBedrockBuildKwargs:
|
||||
|
||||
def test_basic_kwargs(self, transport):
|
||||
msgs = [{"role": "user", "content": "Hello"}]
|
||||
kw = transport.build_kwargs(model="anthropic.claude-3-5-sonnet-20241022-v2:0", messages=msgs)
|
||||
assert kw["modelId"] == "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
assert kw["__bedrock_converse__"] is True
|
||||
assert kw["__bedrock_region__"] == "us-east-1"
|
||||
assert "messages" in kw
|
||||
|
||||
def test_custom_region(self, transport):
|
||||
msgs = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=msgs,
|
||||
region="eu-west-1",
|
||||
)
|
||||
assert kw["__bedrock_region__"] == "eu-west-1"
|
||||
|
||||
def test_max_tokens(self, transport):
|
||||
msgs = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=msgs,
|
||||
max_tokens=8192,
|
||||
)
|
||||
assert kw["inferenceConfig"]["maxTokens"] == 8192
|
||||
|
||||
|
||||
class TestBedrockConvertTools:
|
||||
|
||||
def test_convert_tools(self, transport):
|
||||
tools = [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"description": "Run commands",
|
||||
"parameters": {"type": "object", "properties": {"command": {"type": "string"}}},
|
||||
}
|
||||
}]
|
||||
result = transport.convert_tools(tools)
|
||||
assert len(result) == 1
|
||||
assert result[0]["toolSpec"]["name"] == "terminal"
|
||||
|
||||
|
||||
class TestBedrockValidate:
|
||||
|
||||
def test_none(self, transport):
|
||||
assert transport.validate_response(None) is False
|
||||
|
||||
def test_raw_dict_valid(self, transport):
|
||||
assert transport.validate_response({"output": {"message": {}}}) is True
|
||||
|
||||
def test_raw_dict_invalid(self, transport):
|
||||
assert transport.validate_response({"error": "fail"}) is False
|
||||
|
||||
def test_normalized_valid(self, transport):
|
||||
r = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="hi"))])
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
|
||||
class TestBedrockMapFinishReason:
|
||||
|
||||
def test_end_turn(self, transport):
|
||||
assert transport.map_finish_reason("end_turn") == "stop"
|
||||
|
||||
def test_tool_use(self, transport):
|
||||
assert transport.map_finish_reason("tool_use") == "tool_calls"
|
||||
|
||||
def test_max_tokens(self, transport):
|
||||
assert transport.map_finish_reason("max_tokens") == "length"
|
||||
|
||||
def test_guardrail(self, transport):
|
||||
assert transport.map_finish_reason("guardrail_intervened") == "content_filter"
|
||||
|
||||
def test_unknown(self, transport):
|
||||
assert transport.map_finish_reason("unknown") == "stop"
|
||||
|
||||
|
||||
class TestBedrockNormalize:
|
||||
|
||||
def _make_bedrock_response(self, text="Hello", tool_calls=None, stop_reason="end_turn"):
|
||||
"""Build a raw Bedrock converse response dict."""
|
||||
content = []
|
||||
if text:
|
||||
content.append({"text": text})
|
||||
if tool_calls:
|
||||
for tc in tool_calls:
|
||||
content.append({
|
||||
"toolUse": {
|
||||
"toolUseId": tc["id"],
|
||||
"name": tc["name"],
|
||||
"input": tc["input"],
|
||||
}
|
||||
})
|
||||
return {
|
||||
"output": {"message": {"role": "assistant", "content": content}},
|
||||
"stopReason": stop_reason,
|
||||
"usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},
|
||||
}
|
||||
|
||||
def test_text_response(self, transport):
|
||||
raw = self._make_bedrock_response(text="Hello world")
|
||||
nr = transport.normalize_response(raw)
|
||||
assert isinstance(nr, NormalizedResponse)
|
||||
assert nr.content == "Hello world"
|
||||
assert nr.finish_reason == "stop"
|
||||
|
||||
def test_tool_call_response(self, transport):
|
||||
raw = self._make_bedrock_response(
|
||||
text=None,
|
||||
tool_calls=[{"id": "tool_1", "name": "terminal", "input": {"command": "ls"}}],
|
||||
stop_reason="tool_use",
|
||||
)
|
||||
nr = transport.normalize_response(raw)
|
||||
assert nr.finish_reason == "tool_calls"
|
||||
assert len(nr.tool_calls) == 1
|
||||
assert nr.tool_calls[0].name == "terminal"
|
||||
|
||||
def test_raw_reasoning_content_response(self, transport):
|
||||
raw = {
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"reasoningContent": {"text": "Let me think..."}},
|
||||
{"text": "Answer."},
|
||||
],
|
||||
}
|
||||
},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},
|
||||
}
|
||||
nr = transport.normalize_response(raw)
|
||||
assert nr.reasoning == "Let me think..."
|
||||
assert nr.content == "Answer."
|
||||
|
||||
def test_already_normalized_response(self, transport):
|
||||
"""Test normalize_response handles already-normalized SimpleNamespace (from dispatch site)."""
|
||||
pre_normalized = SimpleNamespace(
|
||||
choices=[SimpleNamespace(
|
||||
message=SimpleNamespace(
|
||||
content="Hello from Bedrock",
|
||||
tool_calls=None,
|
||||
reasoning=None,
|
||||
reasoning_content=None,
|
||||
),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5, total_tokens=15),
|
||||
)
|
||||
nr = transport.normalize_response(pre_normalized)
|
||||
assert isinstance(nr, NormalizedResponse)
|
||||
assert nr.content == "Hello from Bedrock"
|
||||
assert nr.finish_reason == "stop"
|
||||
assert nr.usage is not None
|
||||
assert nr.usage.prompt_tokens == 10
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,298 @@
|
||||
"""Tests for the optional codex app-server runtime gate.
|
||||
|
||||
These are unit tests for the api_mode rewriter and the wire-level transport
|
||||
module. They do NOT require the `codex` CLI to be installed — that's
|
||||
covered by a separate live test gated on `codex --version`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.runtime_provider import (
|
||||
_VALID_API_MODES,
|
||||
_maybe_apply_codex_app_server_runtime,
|
||||
)
|
||||
|
||||
|
||||
class TestApiModeRegistration:
|
||||
"""The new api_mode must be registered or downstream parsing rejects it."""
|
||||
|
||||
def test_codex_app_server_is_a_valid_api_mode(self) -> None:
|
||||
assert "codex_app_server" in _VALID_API_MODES
|
||||
|
||||
def test_existing_api_modes_still_present(self) -> None:
|
||||
# Regression guard: don't accidentally delete other api_modes when
|
||||
# touching this set.
|
||||
for mode in (
|
||||
"chat_completions",
|
||||
"codex_responses",
|
||||
"anthropic_messages",
|
||||
"bedrock_converse",
|
||||
):
|
||||
assert mode in _VALID_API_MODES
|
||||
|
||||
|
||||
class TestMaybeApplyCodexAppServerRuntime:
|
||||
"""The opt-in helper that rewrites api_mode → codex_app_server."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_cfg",
|
||||
[
|
||||
None,
|
||||
{},
|
||||
{"openai_runtime": ""},
|
||||
{"openai_runtime": "auto"},
|
||||
{"openai_runtime": "AUTO"},
|
||||
{"other_key": "codex_app_server"}, # wrong key
|
||||
],
|
||||
)
|
||||
def test_default_off_for_openai(self, model_cfg) -> None:
|
||||
"""Default behavior is preserved when the flag is unset/auto."""
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai", api_mode="chat_completions", model_cfg=model_cfg
|
||||
)
|
||||
assert got == "chat_completions"
|
||||
|
||||
def test_opt_in_rewrites_openai(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai",
|
||||
api_mode="chat_completions",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
def test_opt_in_rewrites_openai_codex(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai-codex",
|
||||
api_mode="codex_responses",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai",
|
||||
api_mode="chat_completions",
|
||||
model_cfg={"openai_runtime": "Codex_App_Server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider",
|
||||
[
|
||||
"anthropic",
|
||||
"openrouter",
|
||||
"xai",
|
||||
"qwen-oauth",
|
||||
"google-gemini-cli",
|
||||
"opencode-zen",
|
||||
"bedrock",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_other_providers_never_rerouted(self, provider) -> None:
|
||||
"""Non-OpenAI providers MUST NOT be rerouted even with the flag set —
|
||||
codex's app-server can only run OpenAI/Codex auth flows."""
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider=provider,
|
||||
api_mode="anthropic_messages",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "anthropic_messages", (
|
||||
f"provider={provider!r} should not be rerouted to codex_app_server"
|
||||
)
|
||||
|
||||
|
||||
class TestCodexAppServerModule:
|
||||
"""Module-surface tests for the JSON-RPC speaker. Don't require codex CLI."""
|
||||
|
||||
def test_module_imports(self) -> None:
|
||||
from agent.transports import codex_app_server
|
||||
|
||||
assert codex_app_server.MIN_CODEX_VERSION >= (0, 1, 0)
|
||||
assert callable(codex_app_server.parse_codex_version)
|
||||
assert callable(codex_app_server.check_codex_binary)
|
||||
|
||||
def test_parse_codex_version_valid(self) -> None:
|
||||
from agent.transports.codex_app_server import parse_codex_version
|
||||
|
||||
assert parse_codex_version("codex-cli 0.130.0") == (0, 130, 0)
|
||||
assert parse_codex_version("codex-cli 1.2.3 (extra metadata)") == (1, 2, 3)
|
||||
assert parse_codex_version("codex 99.0.1\n") == (99, 0, 1)
|
||||
|
||||
def test_parse_codex_version_invalid(self) -> None:
|
||||
from agent.transports.codex_app_server import parse_codex_version
|
||||
|
||||
assert parse_codex_version("nope") is None
|
||||
assert parse_codex_version("") is None
|
||||
assert parse_codex_version(None) is None # type: ignore[arg-type]
|
||||
|
||||
def test_check_binary_handles_missing_executable(self) -> None:
|
||||
from agent.transports.codex_app_server import check_codex_binary
|
||||
|
||||
ok, msg = check_codex_binary(codex_bin="/nonexistent/codex/binary/path")
|
||||
assert ok is False
|
||||
assert "not found" in msg.lower() or "no such" in msg.lower()
|
||||
|
||||
def test_codex_error_class_is_runtimeerror(self) -> None:
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
err = CodexAppServerError(code=-32600, message="boom")
|
||||
assert isinstance(err, RuntimeError)
|
||||
assert "boom" in str(err)
|
||||
assert "-32600" in str(err)
|
||||
|
||||
|
||||
class TestSpawnEnvIsolation:
|
||||
"""The codex spawn must NOT rewrite HOME — codex's shell tool spawns
|
||||
subprocesses (gh, git, npm, aws, gcloud, ...) that need to find their
|
||||
config in the real user $HOME. CODEX_HOME isolates codex's own state,
|
||||
HOME stays unchanged.
|
||||
|
||||
OpenClaw hit this footgun (openclaw/openclaw#81562) — they were
|
||||
rewriting HOME to a synthetic per-agent dir alongside CODEX_HOME,
|
||||
and then `gh auth status` / git config / etc. all broke inside codex
|
||||
shell calls. We avoid the same bug by only overlaying CODEX_HOME and
|
||||
RUST_LOG on top of os.environ.copy().
|
||||
"""
|
||||
|
||||
def test_spawn_env_preserves_HOME(self, monkeypatch):
|
||||
"""The spawn env must contain the parent process's HOME unchanged.
|
||||
Verifies via a subprocess-monkey-patch."""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
# Provide minimal Popen surface so __init__ doesn't crash
|
||||
# on attribute access during construction.
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
|
||||
client = cas.CodexAppServerClient(codex_bin="codex")
|
||||
client._closed = True # so close() is a no-op
|
||||
|
||||
# The spawn env must have HOME=/users/alice unchanged
|
||||
assert captured["env"].get("HOME") == "/users/alice", (
|
||||
f"HOME got rewritten in codex spawn env: "
|
||||
f"{captured['env'].get('HOME')!r}. Codex's shell tool's "
|
||||
"subprocesses (gh, git, aws, npm) need the user's real HOME."
|
||||
)
|
||||
|
||||
def test_spawn_env_sets_CODEX_HOME_when_provided(self, monkeypatch):
|
||||
"""CODEX_HOME isolation must still work — that's the whole point
|
||||
of the codex_home arg."""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
|
||||
client = cas.CodexAppServerClient(
|
||||
codex_bin="codex", codex_home="/tmp/profile/codex"
|
||||
)
|
||||
client._closed = True
|
||||
|
||||
assert captured["env"].get("CODEX_HOME") == "/tmp/profile/codex"
|
||||
# And HOME still passes through unchanged
|
||||
assert captured["env"].get("HOME") == "/users/alice"
|
||||
|
||||
def test_kanban_worker_adds_only_kanban_writable_root(self, monkeypatch):
|
||||
"""Codex-runtime Kanban workers need to write board state outside
|
||||
their scratch/worktree workspace, but should not fall back to
|
||||
danger-full-access. Hermes passes a narrow app-server config override
|
||||
for the Kanban root only.
|
||||
"""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["cmd"] = list(cmd)
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
monkeypatch.setenv("HERMES_HOME", "/users/alice/.hermes/profiles/backend-worker")
|
||||
monkeypatch.setenv("HERMES_KANBAN_TASK", "t_smoke")
|
||||
monkeypatch.setenv(
|
||||
"HERMES_KANBAN_DB",
|
||||
"/users/alice/.hermes/kanban/boards/smoke/kanban.db",
|
||||
)
|
||||
|
||||
client = cas.CodexAppServerClient(codex_bin="codex")
|
||||
client._closed = True
|
||||
|
||||
cmd = captured["cmd"]
|
||||
assert cmd[:2] == ["codex", "app-server"]
|
||||
assert 'sandbox_mode="workspace-write"' in cmd
|
||||
assert (
|
||||
'sandbox_workspace_write.writable_roots=["/users/alice/.hermes/kanban/boards/smoke"]'
|
||||
in cmd
|
||||
)
|
||||
assert "sandbox_workspace_write.network_access=false" in cmd
|
||||
assert all("danger" not in part for part in cmd)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,302 @@
|
||||
"""Tests for CodexEventProjector — codex item/* events → Hermes messages list.
|
||||
|
||||
Drives projection against fixture notifications captured from codex 0.130.0
|
||||
plus synthetic ones for item types we couldn't auth-test live."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.transports.codex_event_projector import (
|
||||
CodexEventProjector,
|
||||
_deterministic_call_id,
|
||||
_format_tool_args,
|
||||
)
|
||||
|
||||
|
||||
# --- Fixture: real `commandExecution` notification captured from codex 0.130.0
|
||||
COMMAND_EXEC_COMPLETED = {
|
||||
"method": "item/completed",
|
||||
"params": {
|
||||
"item": {
|
||||
"type": "commandExecution",
|
||||
"id": "f8a75c66-a89e-4fd7-8bcf-2d58e664fa9e",
|
||||
"command": "/bin/bash -lc 'echo hello && ls /tmp | head -3'",
|
||||
"cwd": "/tmp",
|
||||
"processId": None,
|
||||
"source": "userShell",
|
||||
"status": "completed",
|
||||
"commandActions": [
|
||||
{"type": "listFiles", "command": "ls /tmp", "path": "tmp"}
|
||||
],
|
||||
"aggregatedOutput": "hello\naa_lang.json\n",
|
||||
"exitCode": 0,
|
||||
"durationMs": 10,
|
||||
},
|
||||
"threadId": "019e1a94-352b-71e1-b214-e5c67c9ec190",
|
||||
"turnId": "019e1a94-3553-7940-8af3-4ca57142deb7",
|
||||
"completedAtMs": 1778562381151,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestProjectionInvariants:
|
||||
"""Universal invariants that must hold across all projection paths."""
|
||||
|
||||
def test_streaming_deltas_dont_materialize(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
for delta_method in (
|
||||
"item/commandExecution/outputDelta",
|
||||
"item/agentMessage/delta",
|
||||
"item/reasoning/delta",
|
||||
):
|
||||
r = p.project({"method": delta_method, "params": {"delta": "x"}})
|
||||
assert r.messages == [], (
|
||||
f"{delta_method} should NOT produce messages — only "
|
||||
f"item/completed materializes"
|
||||
)
|
||||
assert r.is_tool_iteration is False
|
||||
assert r.final_text is None
|
||||
|
||||
def test_turn_started_and_completed_are_silent(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
for method in ("turn/started", "turn/completed", "thread/started"):
|
||||
r = p.project({"method": method, "params": {}})
|
||||
assert r.messages == []
|
||||
|
||||
def test_unknown_method_silent(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project({"method": "totally/unknown", "params": {}})
|
||||
assert r.messages == []
|
||||
|
||||
|
||||
class TestCommandExecutionProjection:
|
||||
"""Real captured notification → assistant tool_call + tool result."""
|
||||
|
||||
def test_command_completed_produces_two_messages(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project(COMMAND_EXEC_COMPLETED)
|
||||
assert len(r.messages) == 2
|
||||
assert r.is_tool_iteration is True
|
||||
|
||||
def test_first_message_is_assistant_tool_call(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assistant = msgs[0]
|
||||
assert assistant["role"] == "assistant"
|
||||
assert assistant["content"] is None
|
||||
assert len(assistant["tool_calls"]) == 1
|
||||
tc = assistant["tool_calls"][0]
|
||||
assert tc["type"] == "function"
|
||||
assert tc["function"]["name"] == "exec_command"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert "echo hello" in args["command"]
|
||||
assert args["cwd"] == "/tmp"
|
||||
|
||||
def test_second_message_is_tool_result_correlating_by_id(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assistant, tool = msgs
|
||||
assert tool["role"] == "tool"
|
||||
assert tool["tool_call_id"] == assistant["tool_calls"][0]["id"]
|
||||
assert "hello" in tool["content"]
|
||||
|
||||
def test_nonzero_exit_code_annotated_in_tool_result(self) -> None:
|
||||
item = {**COMMAND_EXEC_COMPLETED["params"]["item"], "exitCode": 2,
|
||||
"aggregatedOutput": "boom"}
|
||||
notif = {
|
||||
"method": "item/completed",
|
||||
"params": {**COMMAND_EXEC_COMPLETED["params"], "item": item},
|
||||
}
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(notif).messages
|
||||
assert "[exit 2]" in msgs[1]["content"]
|
||||
assert "boom" in msgs[1]["content"]
|
||||
|
||||
def test_deterministic_call_id_across_replay(self) -> None:
|
||||
# Same item id → same call_id (prefix cache must stay valid).
|
||||
p1 = CodexEventProjector()
|
||||
p2 = CodexEventProjector()
|
||||
a = p1.project(COMMAND_EXEC_COMPLETED).messages
|
||||
b = p2.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assert a[0]["tool_calls"][0]["id"] == b[0]["tool_calls"][0]["id"]
|
||||
|
||||
|
||||
class TestAgentMessageProjection:
|
||||
"""assistant text → final_text + assistant message."""
|
||||
|
||||
def test_agent_message_projects_to_assistant(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "agentMessage", "id": "x",
|
||||
"text": "hi there"}},
|
||||
})
|
||||
assert r.final_text == "hi there"
|
||||
assert r.messages == [{"role": "assistant", "content": "hi there"}]
|
||||
assert r.is_tool_iteration is False
|
||||
|
||||
def test_pending_reasoning_attaches_to_next_assistant_message(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
# First a reasoning item lands
|
||||
r1 = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "reasoning", "id": "r1",
|
||||
"summary": ["thinking..."],
|
||||
"content": ["step 1", "step 2"]}},
|
||||
})
|
||||
assert r1.messages == [] # reasoning alone produces no message
|
||||
# Then the assistant message
|
||||
r2 = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "agentMessage", "id": "a1",
|
||||
"text": "ok"}},
|
||||
})
|
||||
assistant = r2.messages[0]
|
||||
assert "reasoning" in assistant
|
||||
assert "thinking" in assistant["reasoning"]
|
||||
assert "step 1" in assistant["reasoning"]
|
||||
|
||||
def test_reasoning_consumed_after_attaching(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "reasoning", "id": "r1", "summary": ["once"], "content": []}}})
|
||||
first = p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "agentMessage", "id": "a", "text": "first"}}}).messages[0]
|
||||
second = p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "agentMessage", "id": "b", "text": "second"}}}).messages[0]
|
||||
assert "reasoning" in first
|
||||
assert "reasoning" not in second
|
||||
|
||||
|
||||
class TestFileChangeProjection:
|
||||
def test_file_change_summary_no_inlined_content(self) -> None:
|
||||
item = {
|
||||
"type": "fileChange",
|
||||
"id": "fc1",
|
||||
"status": "applied",
|
||||
"changes": [
|
||||
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
|
||||
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
|
||||
],
|
||||
}
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project({"method": "item/completed",
|
||||
"params": {"item": item}}).messages
|
||||
assert len(msgs) == 2
|
||||
tc = msgs[0]["tool_calls"][0]
|
||||
assert tc["function"]["name"] == "apply_patch"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert len(args["changes"]) == 2
|
||||
assert all("kind" in c and "path" in c for c in args["changes"])
|
||||
assert "applied" in msgs[1]["content"]
|
||||
|
||||
|
||||
class TestMcpToolCallProjection:
|
||||
def test_mcp_tool_call_namespaced(self) -> None:
|
||||
item = {
|
||||
"type": "mcpToolCall",
|
||||
"id": "m1",
|
||||
"server": "obsidian",
|
||||
"tool": "search_notes",
|
||||
"status": "completed",
|
||||
"arguments": {"query": "hermes"},
|
||||
"result": {"content": [{"text": "found"}]},
|
||||
"error": None,
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert msgs[0]["tool_calls"][0]["function"]["name"] == "mcp.obsidian.search_notes"
|
||||
assert "found" in msgs[1]["content"]
|
||||
|
||||
def test_mcp_error_surfaced(self) -> None:
|
||||
item = {
|
||||
"type": "mcpToolCall", "id": "m2",
|
||||
"server": "x", "tool": "y", "status": "failed",
|
||||
"arguments": {}, "result": None,
|
||||
"error": {"code": -1, "message": "no"},
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert "error" in msgs[1]["content"]
|
||||
|
||||
|
||||
class TestUserAndOpaqueProjection:
|
||||
def test_user_message_text_fragments_only(self) -> None:
|
||||
item = {
|
||||
"type": "userMessage", "id": "u1",
|
||||
"content": [
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "url": "http://x/y"},
|
||||
{"type": "text", "text": "world"},
|
||||
],
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert "hello" in msgs[0]["content"]
|
||||
assert "world" in msgs[0]["content"]
|
||||
|
||||
def test_opaque_item_recorded_without_fabricated_tool_calls(self) -> None:
|
||||
item = {"type": "plan", "id": "p1", "text": "do the thing"}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == "assistant"
|
||||
assert "plan" in msgs[0]["content"].lower()
|
||||
assert "tool_calls" not in msgs[0]
|
||||
|
||||
|
||||
class TestHelpers:
|
||||
def test_deterministic_call_id_stable(self) -> None:
|
||||
assert _deterministic_call_id("exec", "abc") == _deterministic_call_id("exec", "abc")
|
||||
assert _deterministic_call_id("exec", "abc") != _deterministic_call_id("exec", "xyz")
|
||||
|
||||
def test_deterministic_call_id_handles_missing_id(self) -> None:
|
||||
# Should not raise, should be stable for same item type
|
||||
a = _deterministic_call_id("exec", "")
|
||||
b = _deterministic_call_id("exec", "")
|
||||
assert a == b
|
||||
assert "exec" in a
|
||||
|
||||
def test_format_tool_args_sorted_keys(self) -> None:
|
||||
# Sorted keys = deterministic across replays = prefix cache stays valid
|
||||
a = _format_tool_args({"b": 1, "a": 2})
|
||||
b = _format_tool_args({"a": 2, "b": 1})
|
||||
assert a == b
|
||||
|
||||
|
||||
class TestRoleAlternationInvariant:
|
||||
"""The project must never emit two assistant messages back-to-back from
|
||||
one item — that breaks Hermes' message alternation invariant."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
[
|
||||
{"type": "commandExecution", "id": "c1", "command": "x",
|
||||
"cwd": "/", "status": "completed", "aggregatedOutput": "",
|
||||
"exitCode": 0, "commandActions": []},
|
||||
{"type": "fileChange", "id": "f1", "status": "applied",
|
||||
"changes": []},
|
||||
{"type": "mcpToolCall", "id": "m1", "server": "s", "tool": "t",
|
||||
"status": "completed", "arguments": {}, "result": None,
|
||||
"error": None},
|
||||
{"type": "dynamicToolCall", "id": "d1", "tool": "x",
|
||||
"arguments": {}, "status": "completed",
|
||||
"contentItems": [], "success": True},
|
||||
],
|
||||
)
|
||||
def test_tool_items_emit_assistant_then_tool(self, item) -> None:
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert len(msgs) == 2
|
||||
assert msgs[0]["role"] == "assistant"
|
||||
assert msgs[1]["role"] == "tool"
|
||||
assert msgs[1]["tool_call_id"] == msgs[0]["tool_calls"][0]["id"]
|
||||
@@ -0,0 +1,697 @@
|
||||
"""Tests for the ResponsesApiTransport (Codex)."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.transports import get_transport
|
||||
from agent.transports.types import NormalizedResponse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport():
|
||||
import agent.transports.codex # noqa: F401
|
||||
return get_transport("codex_responses")
|
||||
|
||||
|
||||
class TestCodexTransportBasic:
|
||||
|
||||
def test_api_mode(self, transport):
|
||||
assert transport.api_mode == "codex_responses"
|
||||
|
||||
def test_registered_on_import(self, transport):
|
||||
assert transport is not None
|
||||
|
||||
def test_convert_tools(self, transport):
|
||||
tools = [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"description": "Run a command",
|
||||
"parameters": {"type": "object", "properties": {"command": {"type": "string"}}},
|
||||
}
|
||||
}]
|
||||
result = transport.convert_tools(tools)
|
||||
assert len(result) == 1
|
||||
assert result[0]["type"] == "function"
|
||||
assert result[0]["name"] == "terminal"
|
||||
|
||||
|
||||
class TestCodexBuildKwargs:
|
||||
|
||||
def test_basic_kwargs(self, transport):
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4",
|
||||
messages=messages,
|
||||
tools=[],
|
||||
)
|
||||
assert kw["model"] == "gpt-5.4"
|
||||
assert kw["instructions"] == "You are helpful."
|
||||
assert "input" in kw
|
||||
assert kw["store"] is False
|
||||
|
||||
def test_system_extracted_from_messages(self, transport):
|
||||
messages = [
|
||||
{"role": "system", "content": "Custom system prompt"},
|
||||
{"role": "user", "content": "Hi"},
|
||||
]
|
||||
kw = transport.build_kwargs(model="gpt-5.4", messages=messages, tools=[])
|
||||
assert kw["instructions"] == "Custom system prompt"
|
||||
|
||||
def test_no_system_uses_default(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(model="gpt-5.4", messages=messages, tools=[])
|
||||
assert kw["instructions"] # should be non-empty default
|
||||
|
||||
def test_reasoning_config(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning", {}).get("effort") == "high"
|
||||
|
||||
def test_reasoning_disabled(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
reasoning_config={"enabled": False},
|
||||
)
|
||||
assert "reasoning" not in kw or kw.get("include") == []
|
||||
|
||||
def test_session_id_sets_cache_key(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
session_id="test-session-123",
|
||||
)
|
||||
assert kw.get("prompt_cache_key") == "test-session-123"
|
||||
|
||||
def test_github_responses_no_cache_key(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
session_id="test-session",
|
||||
is_github_responses=True,
|
||||
)
|
||||
assert "prompt_cache_key" not in kw
|
||||
|
||||
def test_xai_responses_sends_cache_key_via_extra_body(self, transport):
|
||||
"""xAI's Responses API documents ``prompt_cache_key`` as the
|
||||
body-level cache-routing key (the ``x-grok-conv-id`` header is
|
||||
Chat-Completions-only). Passing it via ``extra_body`` is robust
|
||||
against openai SDK builds whose ``Responses.stream()`` kwarg
|
||||
signature ever drops the field — the body field still serializes
|
||||
and reaches xAI either way. The ``x-grok-conv-id`` header is kept
|
||||
as a belt-and-braces fallback so cache routing survives even
|
||||
when the body field would be stripped by an intermediate proxy.
|
||||
Ref: https://docs.x.ai/developers/advanced-api-usage/prompt-caching/maximizing-cache-hits
|
||||
"""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
session_id="conv-xai-1",
|
||||
is_xai_responses=True,
|
||||
)
|
||||
assert "prompt_cache_key" not in kw
|
||||
assert kw.get("extra_body", {}).get("prompt_cache_key") == "conv-xai-1"
|
||||
assert kw.get("extra_headers", {}).get("x-grok-conv-id") == "conv-xai-1"
|
||||
|
||||
def test_xai_responses_extra_body_preserves_caller_fields(self, transport):
|
||||
"""When the caller already supplies ``extra_body`` (e.g. via
|
||||
request_overrides), the xAI cache-key injection must merge into
|
||||
the existing dict instead of overwriting it. Caller-supplied
|
||||
``prompt_cache_key`` wins (setdefault semantics) so user overrides
|
||||
aren't silently clobbered by the transport."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
session_id="conv-xai-1",
|
||||
is_xai_responses=True,
|
||||
request_overrides={"extra_body": {"prompt_cache_key": "caller-override", "other_field": 42}},
|
||||
)
|
||||
eb = kw.get("extra_body", {})
|
||||
assert eb.get("prompt_cache_key") == "caller-override"
|
||||
assert eb.get("other_field") == 42
|
||||
|
||||
def test_max_tokens(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
max_tokens=4096,
|
||||
)
|
||||
assert kw.get("max_output_tokens") == 4096
|
||||
|
||||
def test_codex_backend_no_max_output_tokens(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
max_tokens=4096,
|
||||
is_codex_backend=True,
|
||||
)
|
||||
assert "max_output_tokens" not in kw
|
||||
|
||||
def test_codex_backend_does_not_set_extra_headers(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4",
|
||||
messages=messages,
|
||||
tools=[],
|
||||
session_id="conv-codex-1",
|
||||
is_codex_backend=True,
|
||||
)
|
||||
|
||||
assert "extra_headers" not in kw
|
||||
|
||||
def test_codex_backend_strips_caller_extra_headers(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4",
|
||||
messages=messages,
|
||||
tools=[],
|
||||
session_id="conv-codex-1",
|
||||
is_codex_backend=True,
|
||||
request_overrides={"extra_headers": {"x-test": "1"}},
|
||||
)
|
||||
|
||||
assert "extra_headers" not in kw
|
||||
|
||||
def test_non_codex_responses_preserves_caller_extra_headers(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4",
|
||||
messages=messages,
|
||||
tools=[],
|
||||
is_codex_backend=False,
|
||||
request_overrides={"extra_headers": {"x-test": "1"}},
|
||||
)
|
||||
|
||||
assert kw["extra_headers"] == {"x-test": "1"}
|
||||
|
||||
def test_xai_headers(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-3", messages=messages, tools=[],
|
||||
session_id="conv-123",
|
||||
is_xai_responses=True,
|
||||
)
|
||||
assert kw.get("extra_headers", {}).get("x-grok-conv-id") == "conv-123"
|
||||
|
||||
def test_xai_headers_preserve_request_override_headers(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-3", messages=messages, tools=[],
|
||||
session_id="conv-123",
|
||||
is_xai_responses=True,
|
||||
request_overrides={"extra_headers": {"X-Test": "1", "X-Trace": "abc"}},
|
||||
)
|
||||
assert kw.get("extra_headers") == {
|
||||
"X-Test": "1",
|
||||
"X-Trace": "abc",
|
||||
"x-grok-conv-id": "conv-123",
|
||||
}
|
||||
|
||||
def test_minimal_effort_clamped(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=messages, tools=[],
|
||||
reasoning_config={"effort": "minimal"},
|
||||
)
|
||||
# "minimal" should be clamped to "low"
|
||||
assert kw.get("reasoning", {}).get("effort") == "low"
|
||||
|
||||
def test_xai_reasoning_effort_passed(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
# xAI Responses receives reasoning.effort on the allowlisted models.
|
||||
assert kw.get("reasoning") == {"effort": "high"}
|
||||
# As of May 2026 (post-revert of PR #26644) we DO request
|
||||
# reasoning.encrypted_content back from xAI so we can replay it
|
||||
# across turns for cross-turn coherence — xAI explicitly relies
|
||||
# on this for their partnership integration. See
|
||||
# tests/run_agent/test_codex_xai_oauth_recovery.py for the
|
||||
# full history.
|
||||
assert "reasoning.encrypted_content" in kw.get("include", [])
|
||||
|
||||
def test_xai_reasoning_disabled_no_reasoning_key(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"enabled": False},
|
||||
)
|
||||
# When reasoning is disabled, do not send the reasoning key at all
|
||||
assert "reasoning" not in kw
|
||||
|
||||
def test_xai_minimal_effort_clamped(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "minimal"},
|
||||
)
|
||||
# "minimal" should be clamped to "low" for xAI as well
|
||||
assert kw.get("reasoning", {}).get("effort") == "low"
|
||||
|
||||
# --- Grok reasoning-effort capability allowlist ---
|
||||
# api.x.ai 400s with "Model X does not support parameter reasoningEffort"
|
||||
# on grok-4 / grok-4-fast / grok-3 / grok-code-fast / grok-4.20-0309-*.
|
||||
# Those models reason natively but don't expose the dial. The transport
|
||||
# must omit the `reasoning` key for them. As of May 2026 we DO request
|
||||
# ``reasoning.encrypted_content`` back from xAI on every model —
|
||||
# see test_xai_reasoning_effort_passed for the rationale.
|
||||
|
||||
def test_xai_grok_4_omits_reasoning_effort(self, transport):
|
||||
"""grok-4 / grok-4-0709 reject reasoning.effort with HTTP 400."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in ("grok-4", "grok-4-0709"):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw, (
|
||||
f"{model} must not receive a reasoning key (xAI rejects it)"
|
||||
)
|
||||
# Even without the effort dial we still ask xAI to echo back
|
||||
# encrypted reasoning content so it can be replayed next turn.
|
||||
assert "reasoning.encrypted_content" in kw.get("include", [])
|
||||
|
||||
def test_xai_grok_4_fast_omits_reasoning_effort(self, transport):
|
||||
"""grok-4-fast and grok-4-1-fast variants reject reasoning.effort."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in (
|
||||
"grok-4-fast-reasoning",
|
||||
"grok-4-fast-non-reasoning",
|
||||
"grok-4-1-fast-reasoning",
|
||||
"grok-4-1-fast-non-reasoning",
|
||||
):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "low"},
|
||||
)
|
||||
assert "reasoning" not in kw, (
|
||||
f"{model} must not receive a reasoning key (xAI rejects it)"
|
||||
)
|
||||
|
||||
def test_xai_grok_3_non_mini_omits_reasoning_effort(self, transport):
|
||||
"""Plain grok-3 rejects reasoning.effort — only grok-3-mini accepts it."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-3", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "medium"},
|
||||
)
|
||||
assert "reasoning" not in kw
|
||||
|
||||
def test_xai_grok_3_mini_keeps_reasoning_effort(self, transport):
|
||||
"""grok-3-mini and -fast variants do accept the effort dial."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in ("grok-3-mini", "grok-3-mini-fast"):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning") == {"effort": "high"}
|
||||
|
||||
def test_xai_grok_4_20_0309_variants_omit_reasoning_effort(self, transport):
|
||||
"""grok-4.20-0309-(non-)reasoning reject the effort dial.
|
||||
|
||||
Counterintuitively, only grok-4.20-multi-agent-0309 accepts it.
|
||||
"""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in ("grok-4.20-0309-reasoning", "grok-4.20-0309-non-reasoning"):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw, f"{model} must not receive reasoning"
|
||||
|
||||
def test_xai_grok_4_20_multi_agent_keeps_reasoning_effort(self, transport):
|
||||
"""grok-4.20-multi-agent-0309 is the one grok-4.20 variant that accepts effort."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.20-multi-agent-0309", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "low"},
|
||||
)
|
||||
assert kw.get("reasoning") == {"effort": "low"}
|
||||
|
||||
def test_xai_grok_code_fast_omits_reasoning_effort(self, transport):
|
||||
"""grok-code-fast-1 rejects reasoning.effort."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-code-fast-1", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw
|
||||
|
||||
def test_xai_aggregator_prefix_stripped(self, transport):
|
||||
"""`x-ai/grok-3-mini` (OpenRouter-style slug) still resolves correctly."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
# Effort-capable
|
||||
kw = transport.build_kwargs(
|
||||
model="x-ai/grok-3-mini", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning") == {"effort": "high"}
|
||||
# Effort-incapable
|
||||
kw = transport.build_kwargs(
|
||||
model="x-ai/grok-4-0709", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw
|
||||
|
||||
|
||||
class TestCodexValidateResponse:
|
||||
|
||||
def test_none_response(self, transport):
|
||||
assert transport.validate_response(None) is False
|
||||
|
||||
def test_empty_output(self, transport):
|
||||
r = SimpleNamespace(output=[], output_text=None)
|
||||
assert transport.validate_response(r) is False
|
||||
|
||||
def test_valid_output(self, transport):
|
||||
r = SimpleNamespace(output=[{"type": "message", "content": []}])
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
def test_output_text_fallback_not_valid(self, transport):
|
||||
"""validate_response is strict — output_text doesn't make it valid.
|
||||
The caller handles output_text fallback with diagnostic logging."""
|
||||
r = SimpleNamespace(output=None, output_text="Some text")
|
||||
assert transport.validate_response(r) is False
|
||||
|
||||
|
||||
class TestCodexMapFinishReason:
|
||||
|
||||
def test_completed(self, transport):
|
||||
assert transport.map_finish_reason("completed") == "stop"
|
||||
|
||||
def test_incomplete(self, transport):
|
||||
assert transport.map_finish_reason("incomplete") == "length"
|
||||
|
||||
def test_failed(self, transport):
|
||||
assert transport.map_finish_reason("failed") == "stop"
|
||||
|
||||
def test_unknown(self, transport):
|
||||
assert transport.map_finish_reason("unknown_status") == "stop"
|
||||
|
||||
|
||||
class TestCodexNormalizeResponse:
|
||||
|
||||
def test_text_response(self, transport):
|
||||
"""Normalize a simple text Codex response."""
|
||||
r = SimpleNamespace(
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[SimpleNamespace(type="output_text", text="Hello world")],
|
||||
status="completed",
|
||||
),
|
||||
],
|
||||
status="completed",
|
||||
incomplete_details=None,
|
||||
usage=SimpleNamespace(input_tokens=10, output_tokens=5,
|
||||
input_tokens_details=None, output_tokens_details=None),
|
||||
)
|
||||
nr = transport.normalize_response(r)
|
||||
assert isinstance(nr, NormalizedResponse)
|
||||
assert nr.content == "Hello world"
|
||||
assert nr.finish_reason == "stop"
|
||||
|
||||
def test_message_items_preserved_in_provider_data(self, transport):
|
||||
"""Codex assistant message item ids/phases must survive transport normalization."""
|
||||
r = SimpleNamespace(
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="message",
|
||||
role="assistant",
|
||||
id="msg_abc",
|
||||
phase="final_answer",
|
||||
content=[SimpleNamespace(type="output_text", text="Hello world")],
|
||||
status="completed",
|
||||
),
|
||||
],
|
||||
status="completed",
|
||||
incomplete_details=None,
|
||||
usage=SimpleNamespace(input_tokens=10, output_tokens=5,
|
||||
input_tokens_details=None, output_tokens_details=None),
|
||||
)
|
||||
nr = transport.normalize_response(r)
|
||||
assert nr.codex_message_items == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Hello world"}],
|
||||
"id": "msg_abc",
|
||||
"phase": "final_answer",
|
||||
}
|
||||
]
|
||||
|
||||
def test_tool_call_response(self, transport):
|
||||
"""Normalize a Codex response with tool calls."""
|
||||
r = SimpleNamespace(
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="function_call",
|
||||
call_id="call_abc123",
|
||||
name="terminal",
|
||||
arguments=json.dumps({"command": "ls"}),
|
||||
id="fc_abc123",
|
||||
status="completed",
|
||||
),
|
||||
],
|
||||
status="completed",
|
||||
incomplete_details=None,
|
||||
usage=SimpleNamespace(input_tokens=10, output_tokens=20,
|
||||
input_tokens_details=None, output_tokens_details=None),
|
||||
)
|
||||
nr = transport.normalize_response(r)
|
||||
assert nr.finish_reason == "tool_calls"
|
||||
assert len(nr.tool_calls) == 1
|
||||
tc = nr.tool_calls[0]
|
||||
assert tc.name == "terminal"
|
||||
assert '"command"' in tc.arguments
|
||||
|
||||
|
||||
|
||||
class TestCodexTransportTimeout:
|
||||
"""Forward per-request timeout from build_kwargs to the SDK kwargs."""
|
||||
|
||||
def test_positive_timeout_preserved(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
timeout=600.0,
|
||||
)
|
||||
assert kw.get("timeout") == 600.0
|
||||
|
||||
def test_zero_timeout_dropped(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
timeout=0,
|
||||
)
|
||||
assert "timeout" not in kw
|
||||
|
||||
def test_none_timeout_omitted(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
timeout=None,
|
||||
)
|
||||
assert "timeout" not in kw
|
||||
|
||||
def test_inf_timeout_dropped(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
timeout=float("inf"),
|
||||
)
|
||||
assert "timeout" not in kw
|
||||
|
||||
def test_bool_timeout_dropped(self, transport):
|
||||
"""``True`` is technically int but must not survive — caller bug guard."""
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
timeout=True,
|
||||
)
|
||||
assert "timeout" not in kw
|
||||
|
||||
def test_request_overrides_can_supply_timeout(self, transport):
|
||||
"""request_overrides["timeout"] is honored when no explicit kwarg passed."""
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
request_overrides={"timeout": 450.0},
|
||||
)
|
||||
assert kw.get("timeout") == 450.0
|
||||
|
||||
|
||||
class TestCodexTransportXaiServiceTierStrip:
|
||||
"""xAI Responses API rejects ``service_tier`` (#28490).
|
||||
|
||||
``resolve_fast_mode_overrides`` only returns ``service_tier`` for
|
||||
OpenAI fast-eligible models, so on paper the field should never
|
||||
reach a Grok request. But ``self.service_tier`` lingers across
|
||||
model switches and can also be set directly via ``agent.service_tier``
|
||||
in config.yaml — both leak paths plumb through ``request_overrides``
|
||||
and would 400 against xAI's ``/v1/responses``.
|
||||
Strip defensively when targeting xAI.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def transport(self):
|
||||
from agent.transports.codex import ResponsesApiTransport
|
||||
return ResponsesApiTransport()
|
||||
|
||||
def test_xai_strips_service_tier_from_request_overrides(self, transport):
|
||||
"""Headline #28490 case: service_tier=priority leaks through
|
||||
request_overrides, must not reach the xAI request body."""
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
is_xai_responses=True,
|
||||
request_overrides={"service_tier": "priority"},
|
||||
)
|
||||
assert "service_tier" not in kw, (
|
||||
f"service_tier must be stripped on xAI requests, "
|
||||
f"got {kw.get('service_tier')!r}"
|
||||
)
|
||||
|
||||
def test_non_xai_codex_preserves_service_tier(self, transport):
|
||||
"""The strip is xAI-only — native Codex DOES accept
|
||||
service_tier=priority (OpenAI Priority Processing). Stripping
|
||||
it elsewhere would silently disable the user's fast-mode opt-in.
|
||||
"""
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
is_xai_responses=False,
|
||||
is_codex_backend=True,
|
||||
request_overrides={"service_tier": "priority"},
|
||||
)
|
||||
assert kw.get("service_tier") == "priority", (
|
||||
"non-xAI codex_responses providers must keep service_tier"
|
||||
)
|
||||
|
||||
def test_github_responses_preserves_service_tier(self, transport):
|
||||
"""GitHub Models (Copilot) is another codex_responses surface
|
||||
that should not be affected by the xAI strip."""
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[],
|
||||
is_github_responses=True,
|
||||
request_overrides={"service_tier": "priority"},
|
||||
)
|
||||
assert kw.get("service_tier") == "priority"
|
||||
|
||||
|
||||
class TestPreflightSlashEnumStrip:
|
||||
"""xAI Responses safety-net: strip slash-containing enum values
|
||||
when the model name indicates a Grok target (#28490).
|
||||
|
||||
Native Codex accepts ``/``-containing enums; xAI rejects them with
|
||||
HTTP 400 "Invalid arguments passed to the model". The main agent
|
||||
loop and the auxiliary client already sanitize at request-build
|
||||
time; this preflight catches any future code path that bypasses
|
||||
those — gated on model name so we don't unnecessarily strip on
|
||||
non-xAI providers.
|
||||
"""
|
||||
|
||||
def _make_kwargs(self, model: str, enum_values: list[str]) -> dict:
|
||||
return {
|
||||
"model": model,
|
||||
"instructions": "test",
|
||||
"input": [{"role": "user", "content": "hi"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "pick_model",
|
||||
"description": "pick a model",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model_id": {
|
||||
"type": "string",
|
||||
"enum": enum_values,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def test_grok_model_strips_slash_enum_values(self):
|
||||
"""When the model name is Grok-family, slash-containing enum
|
||||
values are stripped so xAI doesn't 400 on the tool schema."""
|
||||
from agent.codex_responses_adapter import _preflight_codex_api_kwargs
|
||||
kwargs = self._make_kwargs(
|
||||
"grok-4.3",
|
||||
["Qwen/Qwen3.5-0.8B", "openai/gpt-oss-20b", "plain-id"],
|
||||
)
|
||||
result = _preflight_codex_api_kwargs(kwargs)
|
||||
# The enum keyword itself is stripped (per strip_slash_enum's
|
||||
# semantics — it removes the constraint entirely when any value
|
||||
# contains /).
|
||||
params = result["tools"][0]["parameters"]
|
||||
assert "enum" not in params["properties"]["model_id"], (
|
||||
"slash-containing enum must be stripped on Grok"
|
||||
)
|
||||
|
||||
def test_aggregator_prefixed_grok_also_strips(self):
|
||||
"""Aggregator-prefixed (x-ai/grok-*) names hit the same path."""
|
||||
from agent.codex_responses_adapter import _preflight_codex_api_kwargs
|
||||
kwargs = self._make_kwargs(
|
||||
"x-ai/grok-4.3",
|
||||
["Qwen/Qwen3.5-0.8B"],
|
||||
)
|
||||
result = _preflight_codex_api_kwargs(kwargs)
|
||||
assert "enum" not in result["tools"][0]["parameters"]["properties"]["model_id"]
|
||||
|
||||
def test_non_grok_model_preserves_slash_enum_values(self):
|
||||
"""Native Codex / GitHub Models DO accept slash-containing
|
||||
enums. The safety-net must NOT strip there or we silently
|
||||
degrade tool-schema constraints on every codex_responses
|
||||
provider that isn't xAI."""
|
||||
from agent.codex_responses_adapter import _preflight_codex_api_kwargs
|
||||
kwargs = self._make_kwargs(
|
||||
"gpt-5.5",
|
||||
["Qwen/Qwen3.5-0.8B", "plain-id"],
|
||||
)
|
||||
result = _preflight_codex_api_kwargs(kwargs)
|
||||
params = result["tools"][0]["parameters"]
|
||||
# The enum must survive on non-xAI providers.
|
||||
assert params["properties"]["model_id"].get("enum") == [
|
||||
"Qwen/Qwen3.5-0.8B", "plain-id"
|
||||
]
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Tests for the hermes-tools-as-MCP server module surface.
|
||||
|
||||
We don't run a live MCP session in unit tests — that requires the codex
|
||||
subprocess + client + an event loop. These tests pin the static
|
||||
contract: the module imports, the EXPOSED_TOOLS list is sane, and the
|
||||
build helper assembles a server when the SDK is present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
|
||||
|
||||
class TestModuleSurface:
|
||||
def test_module_imports_clean(self):
|
||||
from agent.transports import hermes_tools_mcp_server as m
|
||||
assert callable(m.main)
|
||||
assert callable(m._build_server)
|
||||
assert isinstance(m.EXPOSED_TOOLS, tuple)
|
||||
assert len(m.EXPOSED_TOOLS) > 0
|
||||
|
||||
def test_exposed_tools_are_safe_subset(self):
|
||||
"""We MUST NOT expose tools codex already has, because codex'
|
||||
own builtins are better-integrated with its sandbox + approvals.
|
||||
Specifically: no terminal/shell, no read_file/write_file, no
|
||||
patch — those are codex's built-in tools."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
forbidden = {
|
||||
"terminal", "shell", "read_file", "write_file", "patch",
|
||||
"search_files", "process",
|
||||
}
|
||||
leaked = forbidden & set(EXPOSED_TOOLS)
|
||||
assert not leaked, (
|
||||
f"these tools must NOT be exposed via the codex callback "
|
||||
f"because codex has built-in equivalents: {leaked}"
|
||||
)
|
||||
|
||||
def test_expected_hermes_specific_tools_listed(self):
|
||||
"""The Hermes-specific tools should be present so users on the
|
||||
codex runtime keep access to them."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for required in (
|
||||
"web_search",
|
||||
"web_extract",
|
||||
"browser_navigate",
|
||||
"vision_analyze",
|
||||
"image_generate",
|
||||
"skill_view",
|
||||
):
|
||||
assert required in EXPOSED_TOOLS, f"missing {required!r}"
|
||||
|
||||
def test_agent_loop_tools_not_exposed(self):
|
||||
"""delegate_task / memory / session_search / todo require the
|
||||
running AIAgent context to dispatch, so a stateless MCP callback
|
||||
can't drive them. They must NOT be in EXPOSED_TOOLS."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for agent_loop_tool in ("delegate_task", "memory", "session_search", "todo"):
|
||||
assert agent_loop_tool not in EXPOSED_TOOLS, (
|
||||
f"{agent_loop_tool!r} requires the agent loop context "
|
||||
"and can't be reached through a stateless MCP callback"
|
||||
)
|
||||
|
||||
def test_kanban_worker_tools_exposed(self):
|
||||
"""Kanban workers run as `hermes chat -q` subprocesses; if they
|
||||
come up on the codex_app_server runtime, the worker can do the
|
||||
actual work via codex's shell but needs the kanban tools through
|
||||
the MCP callback to report back to the kernel. Without these
|
||||
tools available, the worker would hang at completion time."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
# Worker handoff tools — every dispatched worker uses at least
|
||||
# one of {complete, block, comment} to close out its task.
|
||||
for worker_tool in (
|
||||
"kanban_complete",
|
||||
"kanban_block",
|
||||
"kanban_comment",
|
||||
"kanban_heartbeat",
|
||||
):
|
||||
assert worker_tool in EXPOSED_TOOLS, (
|
||||
f"{worker_tool!r} missing from codex callback — kanban "
|
||||
"workers on codex_app_server runtime would hang"
|
||||
)
|
||||
|
||||
def test_kanban_orchestrator_tools_exposed(self):
|
||||
"""Orchestrator agents need to dispatch new tasks, query the
|
||||
board, and unblock/link tasks. Exposed so an orchestrator on
|
||||
codex_app_server can do its job."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for orch_tool in (
|
||||
"kanban_create",
|
||||
"kanban_show",
|
||||
"kanban_list",
|
||||
"kanban_unblock",
|
||||
"kanban_link",
|
||||
):
|
||||
assert orch_tool in EXPOSED_TOOLS, (
|
||||
f"{orch_tool!r} missing from codex callback"
|
||||
)
|
||||
|
||||
|
||||
class TestMain:
|
||||
def test_main_returns_2_when_mcp_unavailable(self, monkeypatch):
|
||||
"""When the mcp package isn't installed, main() should exit
|
||||
cleanly with code 2 and an install hint, not crash."""
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
def boom_build(*a, **kw):
|
||||
raise ImportError("mcp not installed")
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", boom_build)
|
||||
rc = m.main(["--verbose"])
|
||||
assert rc == 2
|
||||
|
||||
def test_main_handles_keyboard_interrupt(self, monkeypatch):
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
class FakeServer:
|
||||
def run(self):
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", lambda: FakeServer())
|
||||
rc = m.main([])
|
||||
assert rc == 0
|
||||
|
||||
def test_main_returns_1_on_runtime_error(self, monkeypatch):
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
class CrashingServer:
|
||||
def run(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", lambda: CrashingServer())
|
||||
rc = m.main([])
|
||||
assert rc == 1
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for the transport ABC, registry, and AnthropicTransport."""
|
||||
|
||||
import pytest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.transports.base import ProviderTransport
|
||||
from agent.transports.types import NormalizedResponse
|
||||
from agent.transports import get_transport, register_transport, _REGISTRY
|
||||
|
||||
|
||||
# ── ABC contract tests ──────────────────────────────────────────────────
|
||||
|
||||
class TestProviderTransportABC:
|
||||
"""Verify the ABC contract is enforceable."""
|
||||
|
||||
def test_cannot_instantiate_abc(self):
|
||||
with pytest.raises(TypeError):
|
||||
ProviderTransport()
|
||||
|
||||
def test_concrete_must_implement_all_abstract(self):
|
||||
class Incomplete(ProviderTransport):
|
||||
@property
|
||||
def api_mode(self):
|
||||
return "test"
|
||||
with pytest.raises(TypeError):
|
||||
Incomplete()
|
||||
|
||||
def test_minimal_concrete(self):
|
||||
class Minimal(ProviderTransport):
|
||||
@property
|
||||
def api_mode(self):
|
||||
return "test_minimal"
|
||||
def convert_messages(self, messages, **kw):
|
||||
return messages
|
||||
def convert_tools(self, tools):
|
||||
return tools
|
||||
def build_kwargs(self, model, messages, tools=None, **params):
|
||||
return {"model": model, "messages": messages}
|
||||
def normalize_response(self, response, **kw):
|
||||
return NormalizedResponse(content="ok", tool_calls=None, finish_reason="stop")
|
||||
|
||||
t = Minimal()
|
||||
assert t.api_mode == "test_minimal"
|
||||
assert t.validate_response(None) is True # default
|
||||
assert t.extract_cache_stats(None) is None # default
|
||||
assert t.map_finish_reason("end_turn") == "end_turn" # default passthrough
|
||||
|
||||
|
||||
# ── Registry tests ───────────────────────────────────────────────────────
|
||||
|
||||
class TestTransportRegistry:
|
||||
|
||||
def test_get_unregistered_returns_none(self):
|
||||
assert get_transport("nonexistent_mode") is None
|
||||
|
||||
def test_anthropic_registered_on_import(self):
|
||||
import agent.transports.anthropic # noqa: F401
|
||||
t = get_transport("anthropic_messages")
|
||||
assert t is not None
|
||||
assert t.api_mode == "anthropic_messages"
|
||||
|
||||
def test_discovers_missing_transport_when_registry_partially_populated(self):
|
||||
"""Importing one transport directly must not hide other valid api_modes."""
|
||||
import agent.transports.chat_completions # noqa: F401
|
||||
t = get_transport("codex_responses")
|
||||
assert t is not None
|
||||
assert t.api_mode == "codex_responses"
|
||||
|
||||
def test_register_and_get(self):
|
||||
class DummyTransport(ProviderTransport):
|
||||
@property
|
||||
def api_mode(self):
|
||||
return "dummy_test"
|
||||
def convert_messages(self, messages, **kw):
|
||||
return messages
|
||||
def convert_tools(self, tools):
|
||||
return tools
|
||||
def build_kwargs(self, model, messages, tools=None, **params):
|
||||
return {}
|
||||
def normalize_response(self, response, **kw):
|
||||
return NormalizedResponse(content=None, tool_calls=None, finish_reason="stop")
|
||||
|
||||
register_transport("dummy_test", DummyTransport)
|
||||
t = get_transport("dummy_test")
|
||||
assert t.api_mode == "dummy_test"
|
||||
# Cleanup
|
||||
_REGISTRY.pop("dummy_test", None)
|
||||
|
||||
|
||||
# ── AnthropicTransport tests ────────────────────────────────────────────
|
||||
|
||||
class TestAnthropicTransport:
|
||||
|
||||
@pytest.fixture
|
||||
def transport(self):
|
||||
import agent.transports.anthropic # noqa: F401
|
||||
return get_transport("anthropic_messages")
|
||||
|
||||
def test_api_mode(self, transport):
|
||||
assert transport.api_mode == "anthropic_messages"
|
||||
|
||||
def test_convert_tools_simple(self, transport):
|
||||
tools = [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "test_tool",
|
||||
"description": "A test",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
}]
|
||||
result = transport.convert_tools(tools)
|
||||
assert len(result) == 1
|
||||
assert result[0]["name"] == "test_tool"
|
||||
assert "input_schema" in result[0]
|
||||
|
||||
def test_validate_response_none(self, transport):
|
||||
assert transport.validate_response(None) is False
|
||||
|
||||
def test_validate_response_empty_content(self, transport):
|
||||
r = SimpleNamespace(content=[])
|
||||
assert transport.validate_response(r) is False
|
||||
|
||||
def test_validate_response_empty_content_with_end_turn_is_valid(self, transport):
|
||||
r = SimpleNamespace(content=[], stop_reason="end_turn")
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
def test_validate_response_empty_content_with_tool_use_is_invalid(self, transport):
|
||||
r = SimpleNamespace(content=[], stop_reason="tool_use")
|
||||
assert transport.validate_response(r) is False
|
||||
|
||||
def test_validate_response_empty_content_with_refusal_is_valid(self, transport):
|
||||
# Claude 4.5+ returns an empty content list with stop_reason="refusal"
|
||||
# when it declines to respond. It must validate so the response flows
|
||||
# through to normalize_response (which maps refusal → content_filter)
|
||||
# and the loop's refusal handler — instead of being rejected as an
|
||||
# "invalid response" and retried as a deterministic refusal.
|
||||
r = SimpleNamespace(content=[], stop_reason="refusal")
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
def test_validate_response_valid(self, transport):
|
||||
r = SimpleNamespace(content=[SimpleNamespace(type="text", text="hello")])
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
def test_map_finish_reason(self, transport):
|
||||
assert transport.map_finish_reason("end_turn") == "stop"
|
||||
assert transport.map_finish_reason("tool_use") == "tool_calls"
|
||||
assert transport.map_finish_reason("max_tokens") == "length"
|
||||
assert transport.map_finish_reason("stop_sequence") == "stop"
|
||||
assert transport.map_finish_reason("refusal") == "content_filter"
|
||||
assert transport.map_finish_reason("model_context_window_exceeded") == "length"
|
||||
assert transport.map_finish_reason("unknown") == "stop"
|
||||
|
||||
def test_extract_cache_stats_none_usage(self, transport):
|
||||
r = SimpleNamespace(usage=None)
|
||||
assert transport.extract_cache_stats(r) is None
|
||||
|
||||
def test_extract_cache_stats_with_cache(self, transport):
|
||||
usage = SimpleNamespace(cache_read_input_tokens=100, cache_creation_input_tokens=50)
|
||||
r = SimpleNamespace(usage=usage)
|
||||
result = transport.extract_cache_stats(r)
|
||||
assert result == {"cached_tokens": 100, "creation_tokens": 50}
|
||||
|
||||
def test_extract_cache_stats_zero(self, transport):
|
||||
usage = SimpleNamespace(cache_read_input_tokens=0, cache_creation_input_tokens=0)
|
||||
r = SimpleNamespace(usage=usage)
|
||||
assert transport.extract_cache_stats(r) is None
|
||||
|
||||
def test_normalize_response_text(self, transport):
|
||||
"""Test normalization of a simple text response."""
|
||||
r = SimpleNamespace(
|
||||
content=[SimpleNamespace(type="text", text="Hello world")],
|
||||
stop_reason="end_turn",
|
||||
usage=SimpleNamespace(input_tokens=10, output_tokens=5),
|
||||
model="claude-sonnet-4-6",
|
||||
)
|
||||
nr = transport.normalize_response(r)
|
||||
assert isinstance(nr, NormalizedResponse)
|
||||
assert nr.content == "Hello world"
|
||||
assert nr.tool_calls is None or nr.tool_calls == []
|
||||
assert nr.finish_reason == "stop"
|
||||
|
||||
def test_normalize_response_tool_calls(self, transport):
|
||||
"""Test normalization of a tool-use response."""
|
||||
r = SimpleNamespace(
|
||||
content=[
|
||||
SimpleNamespace(
|
||||
type="tool_use",
|
||||
id="toolu_123",
|
||||
name="terminal",
|
||||
input={"command": "ls"},
|
||||
),
|
||||
],
|
||||
stop_reason="tool_use",
|
||||
usage=SimpleNamespace(input_tokens=10, output_tokens=20),
|
||||
model="claude-sonnet-4-6",
|
||||
)
|
||||
nr = transport.normalize_response(r)
|
||||
assert nr.finish_reason == "tool_calls"
|
||||
assert len(nr.tool_calls) == 1
|
||||
tc = nr.tool_calls[0]
|
||||
assert tc.name == "terminal"
|
||||
assert tc.id == "toolu_123"
|
||||
assert '"command"' in tc.arguments
|
||||
|
||||
def test_normalize_response_thinking(self, transport):
|
||||
"""Test normalization preserves thinking content."""
|
||||
r = SimpleNamespace(
|
||||
content=[
|
||||
SimpleNamespace(type="thinking", thinking="Let me think..."),
|
||||
SimpleNamespace(type="text", text="The answer is 42"),
|
||||
],
|
||||
stop_reason="end_turn",
|
||||
usage=SimpleNamespace(input_tokens=10, output_tokens=15),
|
||||
model="claude-sonnet-4-6",
|
||||
)
|
||||
nr = transport.normalize_response(r)
|
||||
assert nr.content == "The answer is 42"
|
||||
assert nr.reasoning == "Let me think..."
|
||||
|
||||
def test_build_kwargs_returns_dict(self, transport):
|
||||
"""Test build_kwargs produces a usable kwargs dict."""
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="claude-sonnet-4-6",
|
||||
messages=messages,
|
||||
max_tokens=1024,
|
||||
)
|
||||
assert isinstance(kw, dict)
|
||||
assert "model" in kw
|
||||
assert "max_tokens" in kw
|
||||
assert "messages" in kw
|
||||
|
||||
def test_convert_messages_extracts_system(self, transport):
|
||||
"""Test convert_messages separates system from messages."""
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "Hi"},
|
||||
]
|
||||
system, msgs = transport.convert_messages(messages)
|
||||
# System should be extracted
|
||||
assert system is not None
|
||||
# Messages should only have user
|
||||
assert len(msgs) >= 1
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Tests for agent/transports/types.py — dataclass construction + helpers."""
|
||||
|
||||
import json
|
||||
|
||||
from agent.transports.types import (
|
||||
NormalizedResponse,
|
||||
ToolCall,
|
||||
Usage,
|
||||
build_tool_call,
|
||||
map_finish_reason,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ToolCall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestToolCall:
|
||||
def test_basic_construction(self):
|
||||
tc = ToolCall(id="call_abc", name="terminal", arguments='{"cmd": "ls"}')
|
||||
assert tc.id == "call_abc"
|
||||
assert tc.name == "terminal"
|
||||
assert tc.arguments == '{"cmd": "ls"}'
|
||||
assert tc.provider_data is None
|
||||
|
||||
def test_none_id(self):
|
||||
tc = ToolCall(id=None, name="read_file", arguments="{}")
|
||||
assert tc.id is None
|
||||
|
||||
def test_provider_data(self):
|
||||
tc = ToolCall(
|
||||
id="call_x",
|
||||
name="t",
|
||||
arguments="{}",
|
||||
provider_data={"call_id": "call_x", "response_item_id": "fc_x"},
|
||||
)
|
||||
assert tc.provider_data["call_id"] == "call_x"
|
||||
assert tc.provider_data["response_item_id"] == "fc_x"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestUsage:
|
||||
def test_defaults(self):
|
||||
u = Usage()
|
||||
assert u.prompt_tokens == 0
|
||||
assert u.completion_tokens == 0
|
||||
assert u.total_tokens == 0
|
||||
assert u.cached_tokens == 0
|
||||
|
||||
def test_explicit(self):
|
||||
u = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150, cached_tokens=80)
|
||||
assert u.total_tokens == 150
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NormalizedResponse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestNormalizedResponse:
|
||||
def test_text_only(self):
|
||||
r = NormalizedResponse(content="hello", tool_calls=None, finish_reason="stop")
|
||||
assert r.content == "hello"
|
||||
assert r.tool_calls is None
|
||||
assert r.finish_reason == "stop"
|
||||
assert r.reasoning is None
|
||||
assert r.usage is None
|
||||
assert r.provider_data is None
|
||||
|
||||
def test_with_tool_calls(self):
|
||||
tcs = [ToolCall(id="call_1", name="terminal", arguments='{"cmd":"pwd"}')]
|
||||
r = NormalizedResponse(content=None, tool_calls=tcs, finish_reason="tool_calls")
|
||||
assert r.finish_reason == "tool_calls"
|
||||
assert len(r.tool_calls) == 1
|
||||
assert r.tool_calls[0].name == "terminal"
|
||||
|
||||
def test_with_reasoning(self):
|
||||
r = NormalizedResponse(
|
||||
content="answer",
|
||||
tool_calls=None,
|
||||
finish_reason="stop",
|
||||
reasoning="I thought about it",
|
||||
)
|
||||
assert r.reasoning == "I thought about it"
|
||||
|
||||
def test_with_provider_data(self):
|
||||
r = NormalizedResponse(
|
||||
content=None,
|
||||
tool_calls=None,
|
||||
finish_reason="stop",
|
||||
provider_data={"reasoning_details": [{"type": "thinking", "thinking": "hmm"}]},
|
||||
)
|
||||
assert r.provider_data["reasoning_details"][0]["type"] == "thinking"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_tool_call
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBuildToolCall:
|
||||
def test_dict_arguments_serialized(self):
|
||||
tc = build_tool_call(id="call_1", name="terminal", arguments={"cmd": "ls"})
|
||||
assert tc.arguments == json.dumps({"cmd": "ls"})
|
||||
assert tc.provider_data is None
|
||||
|
||||
def test_string_arguments_passthrough(self):
|
||||
tc = build_tool_call(id="call_2", name="read_file", arguments='{"path": "/tmp"}')
|
||||
assert tc.arguments == '{"path": "/tmp"}'
|
||||
|
||||
def test_provider_fields(self):
|
||||
tc = build_tool_call(
|
||||
id="call_3",
|
||||
name="terminal",
|
||||
arguments="{}",
|
||||
call_id="call_3",
|
||||
response_item_id="fc_3",
|
||||
)
|
||||
assert tc.provider_data == {"call_id": "call_3", "response_item_id": "fc_3"}
|
||||
|
||||
def test_none_id(self):
|
||||
tc = build_tool_call(id=None, name="t", arguments="{}")
|
||||
assert tc.id is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# map_finish_reason
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMapFinishReason:
|
||||
ANTHROPIC_MAP = {
|
||||
"end_turn": "stop",
|
||||
"tool_use": "tool_calls",
|
||||
"max_tokens": "length",
|
||||
"stop_sequence": "stop",
|
||||
"refusal": "content_filter",
|
||||
}
|
||||
|
||||
def test_known_reason(self):
|
||||
assert map_finish_reason("end_turn", self.ANTHROPIC_MAP) == "stop"
|
||||
assert map_finish_reason("tool_use", self.ANTHROPIC_MAP) == "tool_calls"
|
||||
assert map_finish_reason("max_tokens", self.ANTHROPIC_MAP) == "length"
|
||||
assert map_finish_reason("refusal", self.ANTHROPIC_MAP) == "content_filter"
|
||||
|
||||
def test_unknown_reason_defaults_to_stop(self):
|
||||
assert map_finish_reason("something_new", self.ANTHROPIC_MAP) == "stop"
|
||||
|
||||
def test_none_reason(self):
|
||||
assert map_finish_reason(None, self.ANTHROPIC_MAP) == "stop"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backward-compat property tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestToolCallBackwardCompat:
|
||||
"""Test duck-typing properties that let ToolCall pass through code expecting
|
||||
the old SimpleNamespace(id, type, function=SimpleNamespace(name, arguments)) shape."""
|
||||
|
||||
def test_type_is_function(self):
|
||||
tc = ToolCall(id="1", name="search", arguments='{"q":"test"}')
|
||||
assert tc.type == "function"
|
||||
|
||||
def test_function_returns_self(self):
|
||||
tc = ToolCall(id="1", name="search", arguments='{"q":"test"}')
|
||||
assert tc.function is tc
|
||||
|
||||
def test_function_name_matches(self):
|
||||
tc = ToolCall(id="1", name="search", arguments='{"q":"test"}')
|
||||
assert tc.function.name == "search"
|
||||
assert tc.function.name == tc.name
|
||||
|
||||
def test_function_arguments_matches(self):
|
||||
tc = ToolCall(id="1", name="search", arguments='{"q":"test"}')
|
||||
assert tc.function.arguments == '{"q":"test"}'
|
||||
assert tc.function.arguments == tc.arguments
|
||||
|
||||
def test_call_id_from_provider_data(self):
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"call_id": "c1"})
|
||||
assert tc.call_id == "c1"
|
||||
|
||||
def test_call_id_none_when_no_provider_data(self):
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data=None)
|
||||
assert tc.call_id is None
|
||||
|
||||
def test_response_item_id_from_provider_data(self):
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"response_item_id": "r1"})
|
||||
assert tc.response_item_id == "r1"
|
||||
|
||||
def test_response_item_id_none_when_missing(self):
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"call_id": "c1"})
|
||||
assert tc.response_item_id is None
|
||||
|
||||
def test_getattr_pattern_matches_agent_loop(self):
|
||||
"""run_agent.py uses getattr(tool_call, 'call_id', None) — verify it works."""
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"call_id": "c1"})
|
||||
assert getattr(tc, "call_id", None) == "c1"
|
||||
tc_no_pd = ToolCall(id="1", name="fn", arguments="{}")
|
||||
assert getattr(tc_no_pd, "call_id", None) is None
|
||||
|
||||
def test_extra_content_from_provider_data(self):
|
||||
"""Gemini thought_signature stored in provider_data is exposed via property."""
|
||||
ec = {"google": {"thought_signature": "SIG_ABC123"}}
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"extra_content": ec})
|
||||
assert tc.extra_content == ec
|
||||
|
||||
def test_extra_content_none_when_no_provider_data(self):
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data=None)
|
||||
assert tc.extra_content is None
|
||||
|
||||
def test_extra_content_none_when_key_absent(self):
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"call_id": "c1"})
|
||||
assert tc.extra_content is None
|
||||
|
||||
def test_extra_content_getattr_pattern(self):
|
||||
"""_build_assistant_message uses getattr(tc, 'extra_content', None).
|
||||
|
||||
This is the exact pattern that was broken before the extra_content
|
||||
property was added — ToolCall lacked the property so getattr always
|
||||
returned None, silently dropping the Gemini thought_signature and
|
||||
causing HTTP 400 on subsequent turns (issue #14488).
|
||||
"""
|
||||
ec = {"google": {"thought_signature": "SIG_ABC123"}}
|
||||
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"extra_content": ec})
|
||||
assert getattr(tc, "extra_content", None) == ec
|
||||
|
||||
tc_no_extra = ToolCall(id="1", name="fn", arguments="{}")
|
||||
assert getattr(tc_no_extra, "extra_content", None) is None
|
||||
|
||||
|
||||
class TestNormalizedResponseBackwardCompat:
|
||||
"""Test properties that replaced _nr_to_assistant_message() shim."""
|
||||
|
||||
def test_reasoning_content_from_provider_data(self):
|
||||
nr = NormalizedResponse(
|
||||
content="hi", tool_calls=None, finish_reason="stop",
|
||||
provider_data={"reasoning_content": "thought process"},
|
||||
)
|
||||
assert nr.reasoning_content == "thought process"
|
||||
|
||||
def test_reasoning_content_none_when_absent(self):
|
||||
nr = NormalizedResponse(content="hi", tool_calls=None, finish_reason="stop")
|
||||
assert nr.reasoning_content is None
|
||||
|
||||
def test_reasoning_details_from_provider_data(self):
|
||||
details = [{"type": "thinking", "thinking": "hmm"}]
|
||||
nr = NormalizedResponse(
|
||||
content="hi", tool_calls=None, finish_reason="stop",
|
||||
provider_data={"reasoning_details": details},
|
||||
)
|
||||
assert nr.reasoning_details == details
|
||||
|
||||
def test_reasoning_details_none_when_no_provider_data(self):
|
||||
nr = NormalizedResponse(
|
||||
content="hi", tool_calls=None, finish_reason="stop",
|
||||
provider_data=None,
|
||||
)
|
||||
assert nr.reasoning_details is None
|
||||
|
||||
def test_codex_reasoning_items_from_provider_data(self):
|
||||
items = ["item1", "item2"]
|
||||
nr = NormalizedResponse(
|
||||
content="hi", tool_calls=None, finish_reason="stop",
|
||||
provider_data={"codex_reasoning_items": items},
|
||||
)
|
||||
assert nr.codex_reasoning_items == items
|
||||
|
||||
def test_codex_reasoning_items_none_when_absent(self):
|
||||
nr = NormalizedResponse(content="hi", tool_calls=None, finish_reason="stop")
|
||||
assert nr.codex_reasoning_items is None
|
||||
|
||||
def test_codex_message_items_from_provider_data(self):
|
||||
items = [{"id": "msg_1", "type": "message"}]
|
||||
nr = NormalizedResponse(
|
||||
content="hi", tool_calls=None, finish_reason="stop",
|
||||
provider_data={"codex_message_items": items},
|
||||
)
|
||||
assert nr.codex_message_items == items
|
||||
|
||||
def test_codex_message_items_none_when_absent(self):
|
||||
nr = NormalizedResponse(content="hi", tool_calls=None, finish_reason="stop")
|
||||
assert nr.codex_message_items is None
|
||||
Reference in New Issue
Block a user