Hermes-agent

This commit is contained in:
Zakaria
2026-06-14 14:30:48 -04:00
commit dac4b88b94
5058 changed files with 1884848 additions and 0 deletions
View File
+118
View File
@@ -0,0 +1,118 @@
"""E2E tests: verify _build_kwargs_from_profile produces correct output.
These tests call _build_kwargs_from_profile on the transport directly,
without importing run_agent (which would cause xdist worker contamination).
"""
import pytest
from agent.transports.chat_completions import ChatCompletionsTransport
from providers import get_provider_profile
@pytest.fixture
def transport():
return ChatCompletionsTransport()
def _msgs():
return [{"role": "user", "content": "hi"}]
class TestNvidiaProfileWiring:
def test_nvidia_gets_default_max_tokens(self, transport):
profile = get_provider_profile("nvidia")
kwargs = transport.build_kwargs(
model="nvidia/llama-3.1-nemotron-70b-instruct",
messages=_msgs(),
tools=None,
provider_profile=profile,
max_tokens=None,
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
timeout=300,
reasoning_config=None,
request_overrides=None,
session_id="test",
ollama_num_ctx=None,
)
# NVIDIA profile sets default_max_tokens=16384
assert kwargs.get("max_tokens") == 16384
def test_nvidia_nim_alias(self, transport):
profile = get_provider_profile("nvidia-nim")
assert profile is not None
assert profile.name == "nvidia"
assert profile.default_max_tokens == 16384
def test_nvidia_model_passed(self, transport):
profile = get_provider_profile("nvidia")
kwargs = transport.build_kwargs(
model="nvidia/test-model",
messages=_msgs(),
tools=None,
provider_profile=profile,
max_tokens=None,
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
timeout=300,
reasoning_config=None,
request_overrides=None,
session_id="test",
ollama_num_ctx=None,
)
assert kwargs["model"] == "nvidia/test-model"
def test_nvidia_messages_passed(self, transport):
profile = get_provider_profile("nvidia")
msgs = _msgs()
kwargs = transport.build_kwargs(
model="nvidia/test",
messages=msgs,
tools=None,
provider_profile=profile,
max_tokens=None,
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
timeout=300,
reasoning_config=None,
request_overrides=None,
session_id="test",
ollama_num_ctx=None,
)
assert kwargs["messages"] == msgs
class TestDeepSeekProfileWiring:
def test_deepseek_no_forced_max_tokens(self, transport):
profile = get_provider_profile("deepseek")
kwargs = transport.build_kwargs(
model="deepseek-chat",
messages=_msgs(),
tools=None,
provider_profile=profile,
max_tokens=None,
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
timeout=300,
reasoning_config=None,
request_overrides=None,
session_id="test",
ollama_num_ctx=None,
)
# DeepSeek has no default_max_tokens
assert kwargs["model"] == "deepseek-chat"
assert kwargs.get("max_tokens") is None or "max_tokens" not in kwargs
def test_deepseek_messages_passed(self, transport):
profile = get_provider_profile("deepseek")
msgs = _msgs()
kwargs = transport.build_kwargs(
model="deepseek-chat",
messages=msgs,
tools=None,
provider_profile=profile,
max_tokens=None,
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
timeout=300,
reasoning_config=None,
request_overrides=None,
session_id="test",
ollama_num_ctx=None,
)
assert kwargs["messages"] == msgs
+155
View File
@@ -0,0 +1,155 @@
"""Tests for the model-providers plugin discovery system.
Verifies that:
1. All bundled providers at plugins/model-providers/<name>/ are discovered
2. User plugins at $HERMES_HOME/plugins/model-providers/<name>/ override bundled
3. plugin.yaml manifests with kind=model-provider are correctly categorized
"""
from __future__ import annotations
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
def _clear_provider_caches():
"""Force providers/__init__.py to re-discover on next list_providers()."""
import providers as _pkg
_pkg._REGISTRY.clear()
_pkg._ALIASES.clear()
_pkg._discovered = False
# Evict any cached plugin modules so the next import re-executes.
for mod in list(sys.modules.keys()):
if (
mod.startswith("plugins.model_providers")
or mod.startswith("_hermes_user_provider")
):
del sys.modules[mod]
def test_bundled_plugins_discovered():
"""Every plugins/model-providers/<name>/ should contain a plugin.yaml + __init__.py."""
plugins_dir = REPO_ROOT / "plugins" / "model-providers"
assert plugins_dir.is_dir(), f"Missing {plugins_dir}"
child_dirs = [c for c in plugins_dir.iterdir() if c.is_dir()]
assert len(child_dirs) >= 28, f"Expected at least 28 provider plugins, found {len(child_dirs)}"
for child in child_dirs:
assert (child / "__init__.py").exists(), f"{child.name} missing __init__.py"
assert (child / "plugin.yaml").exists(), f"{child.name} missing plugin.yaml"
def test_all_profiles_register():
"""After discovery, the registry must contain every bundled provider directory.
This is an invariant — the number of profiles matches the number of plugin
directories, not a hardcoded count. Counts shift when providers are
added/removed; that's expected and shouldn't break CI.
"""
_clear_provider_caches()
from providers import list_providers
plugins_dir = REPO_ROOT / "plugins" / "model-providers"
plugin_dir_count = sum(1 for c in plugins_dir.iterdir() if c.is_dir())
profiles = list_providers()
names = sorted(p.name for p in profiles)
# Some plugin __init__.py files register multiple profiles, so the registry
# count is >= the directory count (never less).
assert len(names) >= plugin_dir_count, (
f"Expected at least {plugin_dir_count} profiles (one per plugin dir), got {len(names)}: {names}"
)
# Spot-check representative providers from different categories
for required in (
"openrouter", "anthropic", "custom", "bedrock", "openai-codex",
"minimax-oauth", "gmi", "xiaomi", "alibaba-coding-plan",
):
assert required in names, f"Missing profile: {required}"
def test_user_plugin_overrides_bundled(tmp_path, monkeypatch):
"""A user plugin with the same name must override the bundled profile."""
# Point HERMES_HOME at a fresh temp dir
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
# get_hermes_home() may be module-cached depending on codebase; ensure the
# env var is the source of truth. Most code paths re-read it each call.
# Drop a user plugin that replaces 'gmi'
user_gmi = hermes_home / "plugins" / "model-providers" / "gmi"
user_gmi.mkdir(parents=True)
(user_gmi / "__init__.py").write_text(
"from providers import register_provider\n"
"from providers.base import ProviderProfile\n"
"\n"
"custom_gmi = ProviderProfile(\n"
' name="gmi",\n'
' aliases=("gmi-user-override-test",),\n'
' env_vars=("GMI_API_KEY",),\n'
' base_url="https://user-override.example.com/v1",\n'
' auth_type="api_key",\n'
")\n"
"register_provider(custom_gmi)\n"
)
(user_gmi / "plugin.yaml").write_text(
"name: gmi-user-override\n"
"kind: model-provider\n"
"version: 0.0.1\n"
"description: Test user override\n"
)
_clear_provider_caches()
from providers import get_provider_profile
gmi = get_provider_profile("gmi")
assert gmi is not None
assert gmi.base_url == "https://user-override.example.com/v1", (
f"User override not applied; got base_url={gmi.base_url!r}"
)
assert "gmi-user-override-test" in gmi.aliases
# Clean up: reset discovery state so other tests see the bundled version
_clear_provider_caches()
def test_general_plugin_manager_skips_model_provider_kind(tmp_path, monkeypatch):
"""The general PluginManager must NOT import model-provider plugins
(providers/__init__.py handles them). It records the manifest only."""
from hermes_cli import plugins as plugin_mod
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
# Create a user-installed plugin with an explicit kind: model-provider.
user_plugin = hermes_home / "plugins" / "test-model-provider"
user_plugin.mkdir(parents=True)
(user_plugin / "plugin.yaml").write_text(
"name: test-model-provider\n"
"kind: model-provider\n"
"version: 0.0.1\n"
)
(user_plugin / "__init__.py").write_text(
# Intentionally broken import — if the general loader tries to
# import this module, the test will fail with ImportError.
"raise AssertionError('model-provider plugins must not be imported by PluginManager')\n"
)
# Fresh manager
manager = plugin_mod.PluginManager()
manager.discover_and_load(force=True)
# The manifest should be recorded but not loaded
loaded = manager._plugins.get("test-model-provider")
assert loaded is not None
assert loaded.manifest.kind == "model-provider"
# No import means the module must NOT be in the plugins list as a loaded one.
# We check that the general loader didn't crash and didn't raise from the
# broken __init__.py.
+296
View File
@@ -0,0 +1,296 @@
"""Profile-path parity tests: verify profile path produces identical output to legacy flags.
Each test calls build_kwargs twice — once with legacy flags, once with provider_profile —
and asserts the output is identical. This catches any behavioral drift between the two paths.
"""
import pytest
from agent.transports.chat_completions import ChatCompletionsTransport
from providers import get_provider_profile
@pytest.fixture
def transport():
return ChatCompletionsTransport()
def _msgs():
return [{"role": "user", "content": "hello"}]
def _max_tokens_fn(n):
return {"max_completion_tokens": n}
class TestNvidiaProfileParity:
def test_max_tokens_match(self, transport):
"""NVIDIA profile sets max_tokens=16384; legacy flag is removed."""
profile = transport.build_kwargs(
model="nvidia/nemotron", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("nvidia"),
max_tokens_param_fn=_max_tokens_fn,
)
assert profile["max_completion_tokens"] == 16384
class TestKimiProfileParity:
def test_temperature_omitted(self, transport):
legacy = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi-coding"), omit_temperature=True,
)
profile = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi"),
)
assert "temperature" not in legacy
assert "temperature" not in profile
def test_max_tokens(self, transport):
legacy = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi-coding"), max_tokens_param_fn=_max_tokens_fn,
)
profile = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi"),
max_tokens_param_fn=_max_tokens_fn,
)
assert profile["max_completion_tokens"] == legacy["max_completion_tokens"] == 32000
def test_thinking_enabled(self, transport):
# xor contract: explicit effort → reasoning_effort only, no thinking.
rc = {"enabled": True, "effort": "high"}
legacy = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi-coding"), reasoning_config=rc,
)
profile = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi"),
reasoning_config=rc,
)
assert profile["reasoning_effort"] == legacy["reasoning_effort"] == "high"
assert "thinking" not in profile.get("extra_body", {})
assert "thinking" not in legacy.get("extra_body", {})
def test_thinking_disabled(self, transport):
rc = {"enabled": False}
legacy = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi-coding"), reasoning_config=rc,
)
profile = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi"),
reasoning_config=rc,
)
assert profile["extra_body"]["thinking"] == legacy["extra_body"]["thinking"]
assert profile["extra_body"]["thinking"]["type"] == "disabled"
assert "reasoning_effort" not in profile
assert "reasoning_effort" not in legacy
def test_reasoning_effort_default(self, transport):
# xor contract: enabled w/o effort → thinking-enabled only, no effort.
rc = {"enabled": True}
legacy = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi-coding"), reasoning_config=rc,
)
profile = transport.build_kwargs(
model="kimi-k2", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("kimi"),
reasoning_config=rc,
)
assert profile["extra_body"]["thinking"] == legacy["extra_body"]["thinking"] == {"type": "enabled"}
assert "reasoning_effort" not in profile
assert "reasoning_effort" not in legacy
class TestOpenRouterProfileParity:
def test_provider_preferences(self, transport):
prefs = {"allow": ["anthropic"]}
legacy = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"), provider_preferences=prefs,
)
profile = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"),
provider_preferences=prefs,
)
assert profile["extra_body"]["provider"] == legacy["extra_body"]["provider"]
def test_reasoning_full_config(self, transport):
rc = {"enabled": True, "effort": "high"}
legacy = transport.build_kwargs(
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"), supports_reasoning=True, reasoning_config=rc,
)
profile = transport.build_kwargs(
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"),
supports_reasoning=True, reasoning_config=rc,
)
assert profile["extra_body"]["reasoning"] == legacy["extra_body"]["reasoning"]
def test_default_reasoning(self, transport):
legacy = transport.build_kwargs(
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"), supports_reasoning=True,
)
profile = transport.build_kwargs(
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"),
supports_reasoning=True,
)
assert profile["extra_body"]["reasoning"] == legacy["extra_body"]["reasoning"]
class TestNousProfileParity:
def test_tags(self, transport):
legacy = transport.build_kwargs(
model="hermes-3", messages=_msgs(), tools=None, provider_profile=get_provider_profile("nous"),
)
profile = transport.build_kwargs(
model="hermes-3", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("nous"),
)
assert profile["extra_body"]["tags"] == legacy["extra_body"]["tags"]
def test_reasoning_omitted_when_disabled(self, transport):
rc = {"enabled": False}
legacy = transport.build_kwargs(
model="hermes-3", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("nous"), supports_reasoning=True, reasoning_config=rc,
)
profile = transport.build_kwargs(
model="hermes-3", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("nous"),
supports_reasoning=True, reasoning_config=rc,
)
assert "reasoning" not in legacy.get("extra_body", {})
assert "reasoning" not in profile.get("extra_body", {})
class TestQwenProfileParity:
def test_max_tokens(self, transport):
legacy = transport.build_kwargs(
model="qwen3.5", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("qwen-oauth"), max_tokens_param_fn=_max_tokens_fn,
)
profile = transport.build_kwargs(
model="qwen3.5", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("qwen"),
max_tokens_param_fn=_max_tokens_fn,
)
assert profile["max_completion_tokens"] == legacy["max_completion_tokens"] == 65536
def test_vl_high_resolution(self, transport):
legacy = transport.build_kwargs(
model="qwen3.5", messages=_msgs(), tools=None, provider_profile=get_provider_profile("qwen-oauth"),
)
profile = transport.build_kwargs(
model="qwen3.5", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("qwen"),
)
assert profile["extra_body"]["vl_high_resolution_images"] == legacy["extra_body"]["vl_high_resolution_images"]
def test_metadata_top_level(self, transport):
meta = {"sessionId": "s123", "promptId": "p456"}
legacy = transport.build_kwargs(
model="qwen3.5", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("qwen-oauth"), qwen_session_metadata=meta,
)
profile = transport.build_kwargs(
model="qwen3.5", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("qwen"),
qwen_session_metadata=meta,
)
assert profile["metadata"] == legacy["metadata"] == meta
assert "metadata" not in profile.get("extra_body", {})
def test_message_preprocessing(self, transport):
"""Qwen profile normalizes string content to list-of-parts."""
msgs = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "hello"},
]
profile = transport.build_kwargs(
model="qwen3.5", messages=msgs, tools=None,
provider_profile=get_provider_profile("qwen"),
)
out_msgs = profile["messages"]
# System message content normalized + cache_control injected
assert isinstance(out_msgs[0]["content"], list)
assert out_msgs[0]["content"][0]["type"] == "text"
assert "cache_control" in out_msgs[0]["content"][-1]
# User message content normalized
assert isinstance(out_msgs[1]["content"], list)
assert out_msgs[1]["content"][0] == {"type": "text", "text": "hello"}
class TestDeveloperRoleParity:
"""Developer role swap must work on BOTH legacy and profile paths."""
def test_legacy_path_swaps_for_gpt5(self, transport):
msgs = [{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "hi"}]
kw = transport.build_kwargs(
model="gpt-5.4", messages=msgs, tools=None,
)
assert kw["messages"][0]["role"] == "developer"
def test_profile_path_swaps_for_gpt5(self, transport):
msgs = [{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "hi"}]
kw = transport.build_kwargs(
model="gpt-5.4", messages=msgs, tools=None,
provider_profile=get_provider_profile("openrouter"),
)
assert kw["messages"][0]["role"] == "developer"
def test_profile_path_no_swap_for_claude(self, transport):
msgs = [{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "hi"}]
kw = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6", messages=msgs, tools=None,
provider_profile=get_provider_profile("openrouter"),
)
assert kw["messages"][0]["role"] == "system"
class TestRequestOverridesParity:
"""request_overrides with extra_body must merge identically on both paths."""
def test_extra_body_override_legacy(self, transport):
kw = transport.build_kwargs(
model="gpt-5.4", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"),
request_overrides={"extra_body": {"custom_key": "custom_val"}},
)
assert kw["extra_body"]["custom_key"] == "custom_val"
def test_extra_body_override_profile(self, transport):
kw = transport.build_kwargs(
model="gpt-5.4", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"),
request_overrides={"extra_body": {"custom_key": "custom_val"}},
)
assert kw["extra_body"]["custom_key"] == "custom_val"
def test_extra_body_override_merges_with_provider_body(self, transport):
"""Override extra_body merges WITH provider extra_body, not replaces."""
from agent.portal_tags import nous_portal_tags
kw = transport.build_kwargs(
model="hermes-3", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("nous"),
request_overrides={"extra_body": {"custom": True}},
)
assert kw["extra_body"]["tags"] == nous_portal_tags() # from profile
assert kw["extra_body"]["custom"] is True # from override
def test_top_level_override(self, transport):
kw = transport.build_kwargs(
model="gpt-5.4", messages=_msgs(), tools=None,
provider_profile=get_provider_profile("openrouter"),
request_overrides={"top_p": 0.9},
)
assert kw["top_p"] == 0.9
+489
View File
@@ -0,0 +1,489 @@
"""Tests for the provider module registry and profiles."""
from providers import get_provider_profile, _REGISTRY
from providers.base import ProviderProfile, OMIT_TEMPERATURE
class TestRegistry:
def test_discovery_populates_registry(self):
p = get_provider_profile("nvidia")
assert p is not None
assert p.name == "nvidia"
def test_alias_lookup(self):
assert get_provider_profile("kimi").name == "kimi-coding"
assert get_provider_profile("moonshot").name == "kimi-coding"
assert get_provider_profile("kimi-coding-cn").name == "kimi-coding-cn"
assert get_provider_profile("or").name == "openrouter"
assert get_provider_profile("nous-portal").name == "nous"
assert get_provider_profile("qwen").name == "qwen-oauth"
assert get_provider_profile("qwen-portal").name == "qwen-oauth"
def test_unknown_provider_returns_none(self):
assert get_provider_profile("nonexistent-provider") is None
def test_all_providers_have_name(self):
get_provider_profile("nvidia") # trigger discovery
for name, profile in _REGISTRY.items():
assert profile.name == name
class TestNvidiaProfile:
def test_max_tokens(self):
p = get_provider_profile("nvidia")
assert p.default_max_tokens == 16384
def test_no_special_temperature(self):
p = get_provider_profile("nvidia")
assert p.fixed_temperature is None
def test_base_url(self):
p = get_provider_profile("nvidia")
assert "nvidia.com" in p.base_url
def test_billing_header_not_profile_wide(self):
p = get_provider_profile("nvidia")
assert p.default_headers == {}
class TestKimiProfile:
def test_temperature_omit(self):
p = get_provider_profile("kimi")
assert p.fixed_temperature is OMIT_TEMPERATURE
def test_max_tokens(self):
p = get_provider_profile("kimi")
assert p.default_max_tokens == 32000
def test_cn_separate_profile(self):
p = get_provider_profile("kimi-coding-cn")
assert p.name == "kimi-coding-cn"
assert p.env_vars == ("KIMI_CN_API_KEY",)
assert "moonshot.cn" in p.base_url
def test_cn_not_alias_of_kimi(self):
kimi = get_provider_profile("kimi-coding")
cn = get_provider_profile("kimi-coding-cn")
assert kimi is not cn
assert kimi.base_url != cn.base_url
def test_thinking_enabled(self):
# xor contract (fix ce4e74b3): an explicit recognized effort sends
# reasoning_effort ONLY — never paired with extra_body.thinking.
p = get_provider_profile("kimi")
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": True, "effort": "high"})
assert tl["reasoning_effort"] == "high"
assert "thinking" not in eb
def test_thinking_disabled(self):
p = get_provider_profile("kimi")
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": False})
assert eb["thinking"] == {"type": "disabled"}
assert "reasoning_effort" not in tl
def test_reasoning_effort_default(self):
# enabled with no effort → thinking toggle only, no top-level effort.
p = get_provider_profile("kimi")
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": True})
assert eb["thinking"] == {"type": "enabled"}
assert "reasoning_effort" not in tl
def test_no_config_defaults(self):
# No reasoning_config → thinking on, server picks depth; no effort.
p = get_provider_profile("kimi")
eb, tl = p.build_api_kwargs_extras(reasoning_config=None)
assert eb["thinking"] == {"type": "enabled"}
assert "reasoning_effort" not in tl
class TestOpenRouterProfile:
def test_extra_body_with_prefs(self):
p = get_provider_profile("openrouter")
body = p.build_extra_body(provider_preferences={"allow": ["anthropic"]})
assert body["provider"] == {"allow": ["anthropic"]}
def test_extra_body_session_id(self):
p = get_provider_profile("openrouter")
body = p.build_extra_body(session_id="test-session-123")
assert body["session_id"] == "test-session-123"
def test_extra_body_no_prefs(self):
p = get_provider_profile("openrouter")
body = p.build_extra_body()
assert body == {}
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=0.65,
)
assert body["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.65}
]
def test_pareto_score_ignored_for_other_models(self):
"""Score has no effect on any other model — plugins block must not appear."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="anthropic/claude-sonnet-4.6",
openrouter_min_coding_score=0.65,
)
assert "plugins" not in body
def test_pareto_score_unset_omits_plugins(self):
"""Empty/None score → no plugins block (router uses its omission default)."""
p = get_provider_profile("openrouter")
for unset in (None, ""):
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=unset,
)
assert "plugins" not in body, f"unset={unset!r}"
def test_pareto_score_out_of_range_dropped(self):
"""Invalid scores are silently dropped — never forwarded to OR."""
p = get_provider_profile("openrouter")
for bad in (1.5, -0.1, "not-a-number"):
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=bad,
)
assert "plugins" not in body, f"bad={bad!r}"
def test_reasoning_full_config(self):
p = get_provider_profile("openrouter")
eb, _ = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
supports_reasoning=True,
)
assert eb["reasoning"] == {"enabled": True, "effort": "high"}
def test_reasoning_disabled_still_passes(self):
"""OpenRouter passes disabled reasoning through (unlike Nous)."""
p = get_provider_profile("openrouter")
eb, _ = p.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
)
assert eb["reasoning"] == {"enabled": False}
def test_reasoning_disable_omitted_for_mandatory_anthropic(self):
"""Reasoning-mandatory Anthropic models (4.6+/fable) reject any disable
form: OpenRouter translates ``reasoning: {enabled: false}`` into
Anthropic's ``thinking: {type: disabled}``, which 400s. The profile must
omit ``reasoning`` so the model falls back to adaptive thinking instead.
"""
p = get_provider_profile("openrouter")
for model in (
"anthropic/claude-fable-5", # new named model
"anthropic/claude-some-future-7", # unknown → default mandatory
"anthropic/claude-opus-4.8",
"anthropic/claude-opus-4.6",
):
for cfg in ({"enabled": False}, {"effort": "none"}):
eb, _ = p.build_api_kwargs_extras(
reasoning_config=cfg,
supports_reasoning=True,
model=model,
)
assert "reasoning" not in eb, (model, cfg, eb)
def test_reasoning_disable_kept_for_legacy_anthropic(self):
"""Older Anthropic models still accept an explicit disable form, so the
profile must keep forwarding it."""
p = get_provider_profile("openrouter")
for model in (
"anthropic/claude-3.7-sonnet",
"anthropic/claude-opus-4.5",
"anthropic/claude-sonnet-4.5",
):
eb, _ = p.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
model=model,
)
assert eb["reasoning"] == {"enabled": False}, (model, eb)
def test_reasoning_disable_kept_for_non_anthropic(self):
"""Non-Anthropic models (DeepSeek, Qwen, …) disable reasoning fine; the
Anthropic-mandatory guard must not touch them."""
p = get_provider_profile("openrouter")
for model in ("deepseek/deepseek-chat", "qwen/qwen3-max", "openai/gpt-5.4"):
eb, _ = p.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
model=model,
)
assert eb["reasoning"] == {"enabled": False}, (model, eb)
def test_reasoning_omitted_for_mandatory_anthropic_even_when_enabled(self):
"""Reasoning-mandatory Anthropic models (4.6+/fable) use adaptive
thinking — OpenRouter ignores reasoning.effort for them, and sending any
reasoning field makes OpenRouter emit thinking.type.disabled on
tool-continuation turns (whose assistant tool_calls carry no thinking
block), 400ing every turn after the first tool call. The profile must
omit reasoning entirely so the model defaults to adaptive.
"""
p = get_provider_profile("openrouter")
for cfg in (
{"enabled": True, "effort": "medium"},
{"enabled": True, "effort": "xhigh"},
{"effort": "high"},
{"enabled": True},
):
eb, _ = p.build_api_kwargs_extras(
reasoning_config=cfg,
supports_reasoning=True,
model="anthropic/claude-fable-5",
)
assert "reasoning" not in eb, (cfg, eb)
def test_default_reasoning(self):
p = get_provider_profile("openrouter")
eb, _ = p.build_api_kwargs_extras(supports_reasoning=True)
assert eb["reasoning"] == {"enabled": True, "effort": "medium"}
def test_grok_session_id_sets_cache_affinity_header(self):
"""OpenRouter + Grok model + session_id => x-grok-conv-id header."""
p = get_provider_profile("openrouter")
_, tl = p.build_api_kwargs_extras(
model="x-ai/grok-4",
session_id="sess-abc123",
)
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-abc123"
def test_grok_xai_prefix_also_supported(self):
"""xai/ prefix (without dash) should also get the header."""
p = get_provider_profile("openrouter")
_, tl = p.build_api_kwargs_extras(
model="xai/grok-3",
session_id="sess-xyz",
)
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-xyz"
def test_non_grok_model_no_affinity_header(self):
"""OpenRouter + non-Grok model => no x-grok-conv-id header."""
p = get_provider_profile("openrouter")
_, tl = p.build_api_kwargs_extras(
model="anthropic/claude-sonnet-4.6",
session_id="sess-abc123",
)
assert "extra_headers" not in tl
assert "x-grok-conv-id" not in tl
def test_grok_without_session_id_no_header(self):
"""Grok model but no session_id => no header (nothing to pin)."""
p = get_provider_profile("openrouter")
_, tl = p.build_api_kwargs_extras(model="x-ai/grok-4")
assert "extra_headers" not in tl
def test_grok_reasoning_and_header_together(self):
"""Reasoning extra_body and Grok header should coexist."""
p = get_provider_profile("openrouter")
eb, tl = p.build_api_kwargs_extras(
model="x-ai/grok-4",
session_id="sess-123",
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "high"},
)
assert eb["reasoning"] == {"enabled": True, "effort": "high"}
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-123"
# --- reasoning-mandatory Anthropic effort → top-level verbosity (#43432) ---
#
# These models (Claude 4.6+ / fable / mythos-class) ignore
# ``reasoning.effort`` and use adaptive thinking. OpenRouter honors the
# requested effort on the top-level ``verbosity`` field instead (maps to
# Anthropic ``output_config.effort``). The profile must route the existing
# ``reasoning_config["effort"]`` there while still NEVER emitting a
# ``reasoning`` field (which would 400 — see #42991). Gate every fixture on
# the real predicate so this stays a behavior contract, not a name snapshot.
@staticmethod
def _is_mandatory(model):
import inspect
p = get_provider_profile("openrouter")
mod = inspect.getmodule(type(p))
return mod._anthropic_reasoning_is_mandatory(model)
def test_mandatory_anthropic_effort_routes_to_verbosity(self):
"""effort set + reasoning enabled → top-level verbosity == effort,
and NO reasoning field in extra_body.
Covers the full real config range produced by
``hermes_constants.parse_reasoning_effort`` —
``VALID_REASONING_EFFORTS = (minimal, low, medium, high, xhigh)``.
"""
p = get_provider_profile("openrouter")
model = "anthropic/claude-fable-5"
assert self._is_mandatory(model) # fixture really is mandatory
for effort in ("minimal", "low", "medium", "high", "xhigh"):
eb, tl = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
supports_reasoning=True,
model=model,
)
assert tl["verbosity"] == effort, (effort, tl)
assert "reasoning" not in eb, (effort, eb)
def test_mandatory_anthropic_effort_without_enabled_key_routes(self):
"""effort present without an explicit ``enabled`` key still routes to
verbosity (enabled defaults to True)."""
p = get_provider_profile("openrouter")
eb, tl = p.build_api_kwargs_extras(
reasoning_config={"effort": "xhigh"},
supports_reasoning=True,
model="anthropic/claude-fable-5",
)
assert tl["verbosity"] == "xhigh"
assert "reasoning" not in eb
def test_mandatory_anthropic_verbosity_is_value_agnostic_passthrough(self):
"""The mapping passes the effort value through verbatim — it must NOT
clamp or whitelist. ``xhigh`` is a real config value; ``max`` is not
producible by ``parse_reasoning_effort`` today but OpenRouter accepts it
for Claude (live-proven in #43432), so a forward value must survive
rather than be silently dropped. The OpenAI SDK type only literals
``low|medium|high`` but it's a TypedDict (no runtime validation), so the
extended scale reaches the wire untouched."""
p = get_provider_profile("openrouter")
for effort in ("xhigh", "max"):
_, tl = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
supports_reasoning=True,
model="anthropic/claude-fable-5",
)
assert tl["verbosity"] == effort
def test_mandatory_anthropic_no_verbosity_when_effort_absent(self):
"""No effort / none / disabled → no verbosity emitted, so the model
keeps its own adaptive default. Still no reasoning field."""
p = get_provider_profile("openrouter")
model = "anthropic/claude-fable-5"
for cfg in (
None,
{},
{"enabled": True},
{"effort": "none"},
{"enabled": True, "effort": "none"},
{"enabled": False, "effort": "high"}, # explicitly disabled wins
):
eb, tl = p.build_api_kwargs_extras(
reasoning_config=cfg,
supports_reasoning=True,
model=model,
)
assert "verbosity" not in tl, (cfg, tl)
assert "reasoning" not in eb, (cfg, eb)
def test_non_mandatory_reasoning_model_unchanged_no_verbosity(self):
"""Non-mandatory reasoning models (DeepSeek, Qwen, GPT) keep getting
``reasoning`` in extra_body and never get a ``verbosity`` field — the
new path must not touch them."""
p = get_provider_profile("openrouter")
for model in ("deepseek/deepseek-chat", "qwen/qwen3-max", "openai/gpt-5.4"):
assert not self._is_mandatory(model) # fixture really is non-mandatory
eb, tl = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
supports_reasoning=True,
model=model,
)
assert eb["reasoning"] == {"enabled": True, "effort": "high"}, (model, eb)
assert "verbosity" not in tl, (model, tl)
def test_mandatory_anthropic_verbosity_coexists_with_grok_header(self):
"""A reasoning-mandatory Anthropic model is never a Grok model, but the
top-level dict must remain a single merged dict — verify the verbosity
path doesn't clobber the extra_headers slot used by Grok affinity."""
p = get_provider_profile("openrouter")
# mandatory anthropic + effort → verbosity, no extra_headers
_, tl = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
supports_reasoning=True,
model="anthropic/claude-fable-5",
)
assert tl == {"verbosity": "high"}
class TestNousProfile:
def test_tags(self):
from agent.portal_tags import nous_portal_tags
p = get_provider_profile("nous")
body = p.build_extra_body()
assert body["tags"] == nous_portal_tags()
def test_auth_type(self):
p = get_provider_profile("nous")
assert p.auth_type == "oauth_device_code"
def test_reasoning_enabled(self):
p = get_provider_profile("nous")
eb, _ = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "medium"},
supports_reasoning=True,
)
assert eb["reasoning"] == {"enabled": True, "effort": "medium"}
def test_reasoning_omitted_when_disabled(self):
p = get_provider_profile("nous")
eb, _ = p.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
)
assert "reasoning" not in eb
class TestQwenProfile:
def test_max_tokens(self):
p = get_provider_profile("qwen-oauth")
assert p.default_max_tokens == 65536
def test_auth_type(self):
p = get_provider_profile("qwen-oauth")
assert p.auth_type == "oauth_external"
def test_extra_body_vl(self):
p = get_provider_profile("qwen-oauth")
body = p.build_extra_body()
assert body["vl_high_resolution_images"] is True
def test_prepare_messages_normalizes_content(self):
p = get_provider_profile("qwen-oauth")
msgs = [
{"role": "system", "content": "Be helpful"},
{"role": "user", "content": "hello"},
]
result = p.prepare_messages(msgs)
# System message: content normalized to list, cache_control on last part
assert isinstance(result[0]["content"], list)
assert result[0]["content"][-1].get("cache_control") == {"type": "ephemeral"}
assert result[0]["content"][-1]["text"] == "Be helpful"
# User message: content normalized to list
assert isinstance(result[1]["content"], list)
assert result[1]["content"][0]["text"] == "hello"
def test_metadata_top_level(self):
p = get_provider_profile("qwen-oauth")
meta = {"sessionId": "s123", "promptId": "p456"}
eb, tl = p.build_api_kwargs_extras(qwen_session_metadata=meta)
assert tl["metadata"] == meta
assert "metadata" not in eb
class TestBaseProfile:
def test_prepare_messages_passthrough(self):
p = ProviderProfile(name="test")
msgs = [{"role": "user", "content": "hi"}]
assert p.prepare_messages(msgs) is msgs
def test_build_extra_body_empty(self):
p = ProviderProfile(name="test")
assert p.build_extra_body() == {}
def test_build_api_kwargs_extras_empty(self):
p = ProviderProfile(name="test")
eb, tl = p.build_api_kwargs_extras()
assert eb == {}
assert tl == {}
+293
View File
@@ -0,0 +1,293 @@
"""Parity tests: pin the exact current transport behavior per provider.
These tests document the flag-based contract between run_agent.py and
ChatCompletionsTransport.build_kwargs(). When the next PR wires profiles
to replace flags, every assertion here must still pass any failure is
a behavioral regression.
"""
import pytest
from agent.transports.chat_completions import ChatCompletionsTransport
from providers import get_provider_profile
@pytest.fixture
def transport():
return ChatCompletionsTransport()
def _simple_messages():
return [{"role": "user", "content": "hello"}]
def _max_tokens_fn(n):
return {"max_completion_tokens": n}
class TestNvidiaParity:
"""NVIDIA NIM: default max_tokens=16384."""
def test_default_max_tokens(self, transport):
"""NVIDIA default max_tokens=16384 comes from profile, not legacy is_nvidia_nim flag."""
from providers import get_provider_profile
profile = get_provider_profile("nvidia")
kw = transport.build_kwargs(
model="nvidia/llama-3.1-nemotron-70b-instruct",
messages=_simple_messages(),
tools=None,
max_tokens_param_fn=_max_tokens_fn,
provider_profile=profile,
)
assert kw["max_completion_tokens"] == 16384
def test_user_max_tokens_overrides(self, transport):
from providers import get_provider_profile
profile = get_provider_profile("nvidia")
kw = transport.build_kwargs(
model="nvidia/llama-3.1-nemotron-70b-instruct",
messages=_simple_messages(),
tools=None,
max_tokens=4096,
max_tokens_param_fn=_max_tokens_fn,
provider_profile=profile,
)
assert kw["max_completion_tokens"] == 4096 # user overrides default
class TestKimiParity:
"""Kimi: OMIT temperature, max_tokens=32000, thinking + reasoning_effort."""
def test_temperature_omitted(self, transport):
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
omit_temperature=True,
)
assert "temperature" not in kw
def test_default_max_tokens(self, transport):
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
max_tokens_param_fn=_max_tokens_fn,
)
assert kw["max_completion_tokens"] == 32000
def test_thinking_enabled(self, transport):
# xor contract (fix ce4e74b3): an explicit recognized effort sends
# reasoning_effort ONLY — never paired with extra_body.thinking.
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
reasoning_config={"enabled": True, "effort": "high"},
)
assert kw.get("reasoning_effort") == "high"
assert "thinking" not in kw.get("extra_body", {})
def test_thinking_enabled_without_effort(self, transport):
# enabled but no effort → fall back to the thinking toggle, no effort.
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
reasoning_config={"enabled": True},
)
assert kw["extra_body"]["thinking"] == {"type": "enabled"}
assert "reasoning_effort" not in kw
def test_thinking_disabled(self, transport):
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
reasoning_config={"enabled": False},
)
assert kw["extra_body"]["thinking"] == {"type": "disabled"}
assert "reasoning_effort" not in kw
def test_reasoning_effort_top_level(self, transport):
"""Kimi reasoning_effort is a TOP-LEVEL api_kwargs key, NOT in extra_body."""
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
reasoning_config={"enabled": True, "effort": "high"},
)
assert kw.get("reasoning_effort") == "high"
assert "reasoning_effort" not in kw.get("extra_body", {})
def test_reasoning_effort_default_no_effort(self, transport):
# xor contract: enabled with no effort falls back to thinking-enabled
# and emits NO top-level reasoning_effort (previously defaulted to
# "medium" alongside thinking — the pairing this fix removes).
kw = transport.build_kwargs(
model="kimi-k2",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("kimi-coding"),
reasoning_config={"enabled": True},
)
assert "reasoning_effort" not in kw
assert kw["extra_body"]["thinking"] == {"type": "enabled"}
class TestOpenRouterParity:
"""OpenRouter: provider preferences, reasoning in extra_body."""
def test_provider_preferences(self, transport):
prefs = {"allow": ["anthropic"], "sort": "price"}
kw = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("openrouter"),
provider_preferences=prefs,
)
assert kw["extra_body"]["provider"] == prefs
def test_reasoning_passes_full_config(self, transport):
"""OpenRouter passes the FULL reasoning_config dict, not just effort."""
rc = {"enabled": True, "effort": "high"}
kw = transport.build_kwargs(
model="deepseek/deepseek-chat",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("openrouter"),
supports_reasoning=True,
reasoning_config=rc,
)
assert kw["extra_body"]["reasoning"] == rc
def test_reasoning_omitted_for_mandatory_anthropic(self, transport):
"""Adaptive-thinking Anthropic models (4.6+/fable) get NO reasoning
field sending one makes OpenRouter emit thinking.type.disabled on
tool-replay turns, which the model 400s on."""
kw = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("openrouter"),
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "high"},
)
assert "reasoning" not in kw.get("extra_body", {})
def test_default_reasoning_when_no_config(self, transport):
"""When supports_reasoning=True but no config, adds default."""
kw = transport.build_kwargs(
model="deepseek/deepseek-chat",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("openrouter"),
supports_reasoning=True,
)
assert kw["extra_body"]["reasoning"] == {"enabled": True, "effort": "medium"}
class TestNousParity:
"""Nous: product tags, reasoning, omit when disabled."""
def test_tags(self, transport):
from agent.portal_tags import nous_portal_tags
kw = transport.build_kwargs(
model="hermes-3-llama-3.1-405b",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("nous"),
)
assert kw["extra_body"]["tags"] == nous_portal_tags()
def test_reasoning_omitted_when_disabled(self, transport):
"""Nous special case: reasoning omitted entirely when disabled."""
kw = transport.build_kwargs(
model="hermes-3-llama-3.1-405b",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("nous"),
supports_reasoning=True,
reasoning_config={"enabled": False},
)
assert "reasoning" not in kw.get("extra_body", {})
def test_reasoning_enabled(self, transport):
rc = {"enabled": True, "effort": "high"}
kw = transport.build_kwargs(
model="hermes-3-llama-3.1-405b",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("nous"),
supports_reasoning=True,
reasoning_config=rc,
)
assert kw["extra_body"]["reasoning"] == rc
class TestQwenParity:
"""Qwen: max_tokens=65536, vl_high_resolution, metadata top-level."""
def test_default_max_tokens(self, transport):
kw = transport.build_kwargs(
model="qwen3.5-plus",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("qwen-oauth"),
max_tokens_param_fn=_max_tokens_fn,
)
assert kw["max_completion_tokens"] == 65536
def test_vl_high_resolution(self, transport):
kw = transport.build_kwargs(
model="qwen3.5-plus",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("qwen-oauth"),
)
assert kw["extra_body"]["vl_high_resolution_images"] is True
def test_metadata_top_level(self, transport):
"""Qwen metadata goes to top-level api_kwargs, NOT extra_body."""
meta = {"sessionId": "s123", "promptId": "p456"}
kw = transport.build_kwargs(
model="qwen3.5-plus",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("qwen-oauth"),
qwen_session_metadata=meta,
)
assert kw["metadata"] == meta
assert "metadata" not in kw.get("extra_body", {})
class TestCustomOllamaParity:
"""Custom/Ollama: num_ctx, thinking controls — now tested via profile."""
def test_ollama_num_ctx(self, transport):
kw = transport.build_kwargs(
model="llama3.1",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("custom"),
ollama_num_ctx=131072,
)
assert kw["extra_body"]["options"]["num_ctx"] == 131072
def test_think_false_when_disabled(self, transport):
kw = transport.build_kwargs(
model="qwen3:72b",
messages=_simple_messages(),
tools=None,
provider_profile=get_provider_profile("custom"),
reasoning_config={"enabled": False, "effort": "none"},
)
assert kw["extra_body"]["think"] is False