Hermes-agent
This commit is contained in:
@@ -0,0 +1,118 @@
|
||||
"""E2E tests: verify _build_kwargs_from_profile produces correct output.
|
||||
|
||||
These tests call _build_kwargs_from_profile on the transport directly,
|
||||
without importing run_agent (which would cause xdist worker contamination).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
from providers import get_provider_profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport():
|
||||
return ChatCompletionsTransport()
|
||||
|
||||
|
||||
def _msgs():
|
||||
return [{"role": "user", "content": "hi"}]
|
||||
|
||||
|
||||
class TestNvidiaProfileWiring:
|
||||
def test_nvidia_gets_default_max_tokens(self, transport):
|
||||
profile = get_provider_profile("nvidia")
|
||||
kwargs = transport.build_kwargs(
|
||||
model="nvidia/llama-3.1-nemotron-70b-instruct",
|
||||
messages=_msgs(),
|
||||
tools=None,
|
||||
provider_profile=profile,
|
||||
max_tokens=None,
|
||||
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
|
||||
timeout=300,
|
||||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
# NVIDIA profile sets default_max_tokens=16384
|
||||
assert kwargs.get("max_tokens") == 16384
|
||||
|
||||
def test_nvidia_nim_alias(self, transport):
|
||||
profile = get_provider_profile("nvidia-nim")
|
||||
assert profile is not None
|
||||
assert profile.name == "nvidia"
|
||||
assert profile.default_max_tokens == 16384
|
||||
|
||||
def test_nvidia_model_passed(self, transport):
|
||||
profile = get_provider_profile("nvidia")
|
||||
kwargs = transport.build_kwargs(
|
||||
model="nvidia/test-model",
|
||||
messages=_msgs(),
|
||||
tools=None,
|
||||
provider_profile=profile,
|
||||
max_tokens=None,
|
||||
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
|
||||
timeout=300,
|
||||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
assert kwargs["model"] == "nvidia/test-model"
|
||||
|
||||
def test_nvidia_messages_passed(self, transport):
|
||||
profile = get_provider_profile("nvidia")
|
||||
msgs = _msgs()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="nvidia/test",
|
||||
messages=msgs,
|
||||
tools=None,
|
||||
provider_profile=profile,
|
||||
max_tokens=None,
|
||||
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
|
||||
timeout=300,
|
||||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
assert kwargs["messages"] == msgs
|
||||
|
||||
|
||||
class TestDeepSeekProfileWiring:
|
||||
def test_deepseek_no_forced_max_tokens(self, transport):
|
||||
profile = get_provider_profile("deepseek")
|
||||
kwargs = transport.build_kwargs(
|
||||
model="deepseek-chat",
|
||||
messages=_msgs(),
|
||||
tools=None,
|
||||
provider_profile=profile,
|
||||
max_tokens=None,
|
||||
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
|
||||
timeout=300,
|
||||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
# DeepSeek has no default_max_tokens
|
||||
assert kwargs["model"] == "deepseek-chat"
|
||||
assert kwargs.get("max_tokens") is None or "max_tokens" not in kwargs
|
||||
|
||||
def test_deepseek_messages_passed(self, transport):
|
||||
profile = get_provider_profile("deepseek")
|
||||
msgs = _msgs()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="deepseek-chat",
|
||||
messages=msgs,
|
||||
tools=None,
|
||||
provider_profile=profile,
|
||||
max_tokens=None,
|
||||
max_tokens_param_fn=lambda x: {"max_tokens": x} if x else {},
|
||||
timeout=300,
|
||||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
assert kwargs["messages"] == msgs
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Tests for the model-providers plugin discovery system.
|
||||
|
||||
Verifies that:
|
||||
1. All bundled providers at plugins/model-providers/<name>/ are discovered
|
||||
2. User plugins at $HERMES_HOME/plugins/model-providers/<name>/ override bundled
|
||||
3. plugin.yaml manifests with kind=model-provider are correctly categorized
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _clear_provider_caches():
|
||||
"""Force providers/__init__.py to re-discover on next list_providers()."""
|
||||
import providers as _pkg
|
||||
_pkg._REGISTRY.clear()
|
||||
_pkg._ALIASES.clear()
|
||||
_pkg._discovered = False
|
||||
# Evict any cached plugin modules so the next import re-executes.
|
||||
for mod in list(sys.modules.keys()):
|
||||
if (
|
||||
mod.startswith("plugins.model_providers")
|
||||
or mod.startswith("_hermes_user_provider")
|
||||
):
|
||||
del sys.modules[mod]
|
||||
|
||||
|
||||
def test_bundled_plugins_discovered():
|
||||
"""Every plugins/model-providers/<name>/ should contain a plugin.yaml + __init__.py."""
|
||||
plugins_dir = REPO_ROOT / "plugins" / "model-providers"
|
||||
assert plugins_dir.is_dir(), f"Missing {plugins_dir}"
|
||||
|
||||
child_dirs = [c for c in plugins_dir.iterdir() if c.is_dir()]
|
||||
assert len(child_dirs) >= 28, f"Expected at least 28 provider plugins, found {len(child_dirs)}"
|
||||
|
||||
for child in child_dirs:
|
||||
assert (child / "__init__.py").exists(), f"{child.name} missing __init__.py"
|
||||
assert (child / "plugin.yaml").exists(), f"{child.name} missing plugin.yaml"
|
||||
|
||||
|
||||
def test_all_profiles_register():
|
||||
"""After discovery, the registry must contain every bundled provider directory.
|
||||
|
||||
This is an invariant — the number of profiles matches the number of plugin
|
||||
directories, not a hardcoded count. Counts shift when providers are
|
||||
added/removed; that's expected and shouldn't break CI.
|
||||
"""
|
||||
_clear_provider_caches()
|
||||
from providers import list_providers
|
||||
|
||||
plugins_dir = REPO_ROOT / "plugins" / "model-providers"
|
||||
plugin_dir_count = sum(1 for c in plugins_dir.iterdir() if c.is_dir())
|
||||
|
||||
profiles = list_providers()
|
||||
names = sorted(p.name for p in profiles)
|
||||
# Some plugin __init__.py files register multiple profiles, so the registry
|
||||
# count is >= the directory count (never less).
|
||||
assert len(names) >= plugin_dir_count, (
|
||||
f"Expected at least {plugin_dir_count} profiles (one per plugin dir), got {len(names)}: {names}"
|
||||
)
|
||||
|
||||
# Spot-check representative providers from different categories
|
||||
for required in (
|
||||
"openrouter", "anthropic", "custom", "bedrock", "openai-codex",
|
||||
"minimax-oauth", "gmi", "xiaomi", "alibaba-coding-plan",
|
||||
):
|
||||
assert required in names, f"Missing profile: {required}"
|
||||
|
||||
|
||||
def test_user_plugin_overrides_bundled(tmp_path, monkeypatch):
|
||||
"""A user plugin with the same name must override the bundled profile."""
|
||||
# Point HERMES_HOME at a fresh temp dir
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
# get_hermes_home() may be module-cached depending on codebase; ensure the
|
||||
# env var is the source of truth. Most code paths re-read it each call.
|
||||
|
||||
# Drop a user plugin that replaces 'gmi'
|
||||
user_gmi = hermes_home / "plugins" / "model-providers" / "gmi"
|
||||
user_gmi.mkdir(parents=True)
|
||||
(user_gmi / "__init__.py").write_text(
|
||||
"from providers import register_provider\n"
|
||||
"from providers.base import ProviderProfile\n"
|
||||
"\n"
|
||||
"custom_gmi = ProviderProfile(\n"
|
||||
' name="gmi",\n'
|
||||
' aliases=("gmi-user-override-test",),\n'
|
||||
' env_vars=("GMI_API_KEY",),\n'
|
||||
' base_url="https://user-override.example.com/v1",\n'
|
||||
' auth_type="api_key",\n'
|
||||
")\n"
|
||||
"register_provider(custom_gmi)\n"
|
||||
)
|
||||
(user_gmi / "plugin.yaml").write_text(
|
||||
"name: gmi-user-override\n"
|
||||
"kind: model-provider\n"
|
||||
"version: 0.0.1\n"
|
||||
"description: Test user override\n"
|
||||
)
|
||||
|
||||
_clear_provider_caches()
|
||||
from providers import get_provider_profile
|
||||
|
||||
gmi = get_provider_profile("gmi")
|
||||
assert gmi is not None
|
||||
assert gmi.base_url == "https://user-override.example.com/v1", (
|
||||
f"User override not applied; got base_url={gmi.base_url!r}"
|
||||
)
|
||||
assert "gmi-user-override-test" in gmi.aliases
|
||||
|
||||
# Clean up: reset discovery state so other tests see the bundled version
|
||||
_clear_provider_caches()
|
||||
|
||||
|
||||
def test_general_plugin_manager_skips_model_provider_kind(tmp_path, monkeypatch):
|
||||
"""The general PluginManager must NOT import model-provider plugins
|
||||
(providers/__init__.py handles them). It records the manifest only."""
|
||||
from hermes_cli import plugins as plugin_mod
|
||||
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
# Create a user-installed plugin with an explicit kind: model-provider.
|
||||
user_plugin = hermes_home / "plugins" / "test-model-provider"
|
||||
user_plugin.mkdir(parents=True)
|
||||
(user_plugin / "plugin.yaml").write_text(
|
||||
"name: test-model-provider\n"
|
||||
"kind: model-provider\n"
|
||||
"version: 0.0.1\n"
|
||||
)
|
||||
(user_plugin / "__init__.py").write_text(
|
||||
# Intentionally broken import — if the general loader tries to
|
||||
# import this module, the test will fail with ImportError.
|
||||
"raise AssertionError('model-provider plugins must not be imported by PluginManager')\n"
|
||||
)
|
||||
|
||||
# Fresh manager
|
||||
manager = plugin_mod.PluginManager()
|
||||
manager.discover_and_load(force=True)
|
||||
|
||||
# The manifest should be recorded but not loaded
|
||||
loaded = manager._plugins.get("test-model-provider")
|
||||
assert loaded is not None
|
||||
assert loaded.manifest.kind == "model-provider"
|
||||
# No import means the module must NOT be in the plugins list as a loaded one.
|
||||
# We check that the general loader didn't crash and didn't raise from the
|
||||
# broken __init__.py.
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Profile-path parity tests: verify profile path produces identical output to legacy flags.
|
||||
|
||||
Each test calls build_kwargs twice — once with legacy flags, once with provider_profile —
|
||||
and asserts the output is identical. This catches any behavioral drift between the two paths.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
from providers import get_provider_profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport():
|
||||
return ChatCompletionsTransport()
|
||||
|
||||
|
||||
def _msgs():
|
||||
return [{"role": "user", "content": "hello"}]
|
||||
|
||||
|
||||
def _max_tokens_fn(n):
|
||||
return {"max_completion_tokens": n}
|
||||
|
||||
|
||||
class TestNvidiaProfileParity:
|
||||
def test_max_tokens_match(self, transport):
|
||||
"""NVIDIA profile sets max_tokens=16384; legacy flag is removed."""
|
||||
profile = transport.build_kwargs(
|
||||
model="nvidia/nemotron", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("nvidia"),
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
assert profile["max_completion_tokens"] == 16384
|
||||
|
||||
|
||||
class TestKimiProfileParity:
|
||||
def test_temperature_omitted(self, transport):
|
||||
legacy = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"), omit_temperature=True,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi"),
|
||||
)
|
||||
assert "temperature" not in legacy
|
||||
assert "temperature" not in profile
|
||||
|
||||
def test_max_tokens(self, transport):
|
||||
legacy = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"), max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi"),
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
assert profile["max_completion_tokens"] == legacy["max_completion_tokens"] == 32000
|
||||
|
||||
def test_thinking_enabled(self, transport):
|
||||
# xor contract: explicit effort → reasoning_effort only, no thinking.
|
||||
rc = {"enabled": True, "effort": "high"}
|
||||
legacy = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"), reasoning_config=rc,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi"),
|
||||
reasoning_config=rc,
|
||||
)
|
||||
assert profile["reasoning_effort"] == legacy["reasoning_effort"] == "high"
|
||||
assert "thinking" not in profile.get("extra_body", {})
|
||||
assert "thinking" not in legacy.get("extra_body", {})
|
||||
|
||||
def test_thinking_disabled(self, transport):
|
||||
rc = {"enabled": False}
|
||||
legacy = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"), reasoning_config=rc,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi"),
|
||||
reasoning_config=rc,
|
||||
)
|
||||
assert profile["extra_body"]["thinking"] == legacy["extra_body"]["thinking"]
|
||||
assert profile["extra_body"]["thinking"]["type"] == "disabled"
|
||||
assert "reasoning_effort" not in profile
|
||||
assert "reasoning_effort" not in legacy
|
||||
|
||||
def test_reasoning_effort_default(self, transport):
|
||||
# xor contract: enabled w/o effort → thinking-enabled only, no effort.
|
||||
rc = {"enabled": True}
|
||||
legacy = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"), reasoning_config=rc,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="kimi-k2", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("kimi"),
|
||||
reasoning_config=rc,
|
||||
)
|
||||
assert profile["extra_body"]["thinking"] == legacy["extra_body"]["thinking"] == {"type": "enabled"}
|
||||
assert "reasoning_effort" not in profile
|
||||
assert "reasoning_effort" not in legacy
|
||||
|
||||
|
||||
class TestOpenRouterProfileParity:
|
||||
def test_provider_preferences(self, transport):
|
||||
prefs = {"allow": ["anthropic"]}
|
||||
legacy = transport.build_kwargs(
|
||||
model="anthropic/claude-sonnet-4.6", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"), provider_preferences=prefs,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="anthropic/claude-sonnet-4.6", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
provider_preferences=prefs,
|
||||
)
|
||||
assert profile["extra_body"]["provider"] == legacy["extra_body"]["provider"]
|
||||
|
||||
def test_reasoning_full_config(self, transport):
|
||||
rc = {"enabled": True, "effort": "high"}
|
||||
legacy = transport.build_kwargs(
|
||||
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"), supports_reasoning=True, reasoning_config=rc,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
supports_reasoning=True, reasoning_config=rc,
|
||||
)
|
||||
assert profile["extra_body"]["reasoning"] == legacy["extra_body"]["reasoning"]
|
||||
|
||||
def test_default_reasoning(self, transport):
|
||||
legacy = transport.build_kwargs(
|
||||
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"), supports_reasoning=True,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="deepseek/deepseek-chat", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert profile["extra_body"]["reasoning"] == legacy["extra_body"]["reasoning"]
|
||||
|
||||
|
||||
class TestNousProfileParity:
|
||||
def test_tags(self, transport):
|
||||
legacy = transport.build_kwargs(
|
||||
model="hermes-3", messages=_msgs(), tools=None, provider_profile=get_provider_profile("nous"),
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="hermes-3", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("nous"),
|
||||
)
|
||||
assert profile["extra_body"]["tags"] == legacy["extra_body"]["tags"]
|
||||
|
||||
def test_reasoning_omitted_when_disabled(self, transport):
|
||||
rc = {"enabled": False}
|
||||
legacy = transport.build_kwargs(
|
||||
model="hermes-3", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("nous"), supports_reasoning=True, reasoning_config=rc,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="hermes-3", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("nous"),
|
||||
supports_reasoning=True, reasoning_config=rc,
|
||||
)
|
||||
assert "reasoning" not in legacy.get("extra_body", {})
|
||||
assert "reasoning" not in profile.get("extra_body", {})
|
||||
|
||||
|
||||
class TestQwenProfileParity:
|
||||
def test_max_tokens(self, transport):
|
||||
legacy = transport.build_kwargs(
|
||||
model="qwen3.5", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("qwen-oauth"), max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="qwen3.5", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("qwen"),
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
assert profile["max_completion_tokens"] == legacy["max_completion_tokens"] == 65536
|
||||
|
||||
def test_vl_high_resolution(self, transport):
|
||||
legacy = transport.build_kwargs(
|
||||
model="qwen3.5", messages=_msgs(), tools=None, provider_profile=get_provider_profile("qwen-oauth"),
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="qwen3.5", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("qwen"),
|
||||
)
|
||||
assert profile["extra_body"]["vl_high_resolution_images"] == legacy["extra_body"]["vl_high_resolution_images"]
|
||||
|
||||
def test_metadata_top_level(self, transport):
|
||||
meta = {"sessionId": "s123", "promptId": "p456"}
|
||||
legacy = transport.build_kwargs(
|
||||
model="qwen3.5", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("qwen-oauth"), qwen_session_metadata=meta,
|
||||
)
|
||||
profile = transport.build_kwargs(
|
||||
model="qwen3.5", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("qwen"),
|
||||
qwen_session_metadata=meta,
|
||||
)
|
||||
assert profile["metadata"] == legacy["metadata"] == meta
|
||||
assert "metadata" not in profile.get("extra_body", {})
|
||||
|
||||
def test_message_preprocessing(self, transport):
|
||||
"""Qwen profile normalizes string content to list-of-parts."""
|
||||
msgs = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "hello"},
|
||||
]
|
||||
profile = transport.build_kwargs(
|
||||
model="qwen3.5", messages=msgs, tools=None,
|
||||
provider_profile=get_provider_profile("qwen"),
|
||||
)
|
||||
out_msgs = profile["messages"]
|
||||
# System message content normalized + cache_control injected
|
||||
assert isinstance(out_msgs[0]["content"], list)
|
||||
assert out_msgs[0]["content"][0]["type"] == "text"
|
||||
assert "cache_control" in out_msgs[0]["content"][-1]
|
||||
# User message content normalized
|
||||
assert isinstance(out_msgs[1]["content"], list)
|
||||
assert out_msgs[1]["content"][0] == {"type": "text", "text": "hello"}
|
||||
|
||||
|
||||
class TestDeveloperRoleParity:
|
||||
"""Developer role swap must work on BOTH legacy and profile paths."""
|
||||
|
||||
def test_legacy_path_swaps_for_gpt5(self, transport):
|
||||
msgs = [{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=msgs, tools=None,
|
||||
)
|
||||
assert kw["messages"][0]["role"] == "developer"
|
||||
|
||||
def test_profile_path_swaps_for_gpt5(self, transport):
|
||||
msgs = [{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=msgs, tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
)
|
||||
assert kw["messages"][0]["role"] == "developer"
|
||||
|
||||
def test_profile_path_no_swap_for_claude(self, transport):
|
||||
msgs = [{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="anthropic/claude-sonnet-4.6", messages=msgs, tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
)
|
||||
assert kw["messages"][0]["role"] == "system"
|
||||
|
||||
|
||||
class TestRequestOverridesParity:
|
||||
"""request_overrides with extra_body must merge identically on both paths."""
|
||||
|
||||
def test_extra_body_override_legacy(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
request_overrides={"extra_body": {"custom_key": "custom_val"}},
|
||||
)
|
||||
assert kw["extra_body"]["custom_key"] == "custom_val"
|
||||
|
||||
def test_extra_body_override_profile(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
request_overrides={"extra_body": {"custom_key": "custom_val"}},
|
||||
)
|
||||
assert kw["extra_body"]["custom_key"] == "custom_val"
|
||||
|
||||
def test_extra_body_override_merges_with_provider_body(self, transport):
|
||||
"""Override extra_body merges WITH provider extra_body, not replaces."""
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
kw = transport.build_kwargs(
|
||||
model="hermes-3", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("nous"),
|
||||
request_overrides={"extra_body": {"custom": True}},
|
||||
)
|
||||
assert kw["extra_body"]["tags"] == nous_portal_tags() # from profile
|
||||
assert kw["extra_body"]["custom"] is True # from override
|
||||
|
||||
def test_top_level_override(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="gpt-5.4", messages=_msgs(), tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
request_overrides={"top_p": 0.9},
|
||||
)
|
||||
assert kw["top_p"] == 0.9
|
||||
@@ -0,0 +1,489 @@
|
||||
"""Tests for the provider module registry and profiles."""
|
||||
|
||||
from providers import get_provider_profile, _REGISTRY
|
||||
from providers.base import ProviderProfile, OMIT_TEMPERATURE
|
||||
|
||||
|
||||
class TestRegistry:
|
||||
def test_discovery_populates_registry(self):
|
||||
p = get_provider_profile("nvidia")
|
||||
assert p is not None
|
||||
assert p.name == "nvidia"
|
||||
|
||||
def test_alias_lookup(self):
|
||||
assert get_provider_profile("kimi").name == "kimi-coding"
|
||||
assert get_provider_profile("moonshot").name == "kimi-coding"
|
||||
assert get_provider_profile("kimi-coding-cn").name == "kimi-coding-cn"
|
||||
assert get_provider_profile("or").name == "openrouter"
|
||||
assert get_provider_profile("nous-portal").name == "nous"
|
||||
assert get_provider_profile("qwen").name == "qwen-oauth"
|
||||
assert get_provider_profile("qwen-portal").name == "qwen-oauth"
|
||||
|
||||
def test_unknown_provider_returns_none(self):
|
||||
assert get_provider_profile("nonexistent-provider") is None
|
||||
|
||||
def test_all_providers_have_name(self):
|
||||
get_provider_profile("nvidia") # trigger discovery
|
||||
for name, profile in _REGISTRY.items():
|
||||
assert profile.name == name
|
||||
|
||||
|
||||
class TestNvidiaProfile:
|
||||
def test_max_tokens(self):
|
||||
p = get_provider_profile("nvidia")
|
||||
assert p.default_max_tokens == 16384
|
||||
|
||||
def test_no_special_temperature(self):
|
||||
p = get_provider_profile("nvidia")
|
||||
assert p.fixed_temperature is None
|
||||
|
||||
def test_base_url(self):
|
||||
p = get_provider_profile("nvidia")
|
||||
assert "nvidia.com" in p.base_url
|
||||
|
||||
def test_billing_header_not_profile_wide(self):
|
||||
p = get_provider_profile("nvidia")
|
||||
assert p.default_headers == {}
|
||||
|
||||
|
||||
class TestKimiProfile:
|
||||
def test_temperature_omit(self):
|
||||
p = get_provider_profile("kimi")
|
||||
assert p.fixed_temperature is OMIT_TEMPERATURE
|
||||
|
||||
def test_max_tokens(self):
|
||||
p = get_provider_profile("kimi")
|
||||
assert p.default_max_tokens == 32000
|
||||
|
||||
def test_cn_separate_profile(self):
|
||||
p = get_provider_profile("kimi-coding-cn")
|
||||
assert p.name == "kimi-coding-cn"
|
||||
assert p.env_vars == ("KIMI_CN_API_KEY",)
|
||||
assert "moonshot.cn" in p.base_url
|
||||
|
||||
def test_cn_not_alias_of_kimi(self):
|
||||
kimi = get_provider_profile("kimi-coding")
|
||||
cn = get_provider_profile("kimi-coding-cn")
|
||||
assert kimi is not cn
|
||||
assert kimi.base_url != cn.base_url
|
||||
|
||||
def test_thinking_enabled(self):
|
||||
# xor contract (fix ce4e74b3): an explicit recognized effort sends
|
||||
# reasoning_effort ONLY — never paired with extra_body.thinking.
|
||||
p = get_provider_profile("kimi")
|
||||
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": True, "effort": "high"})
|
||||
assert tl["reasoning_effort"] == "high"
|
||||
assert "thinking" not in eb
|
||||
|
||||
def test_thinking_disabled(self):
|
||||
p = get_provider_profile("kimi")
|
||||
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": False})
|
||||
assert eb["thinking"] == {"type": "disabled"}
|
||||
assert "reasoning_effort" not in tl
|
||||
|
||||
def test_reasoning_effort_default(self):
|
||||
# enabled with no effort → thinking toggle only, no top-level effort.
|
||||
p = get_provider_profile("kimi")
|
||||
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": True})
|
||||
assert eb["thinking"] == {"type": "enabled"}
|
||||
assert "reasoning_effort" not in tl
|
||||
|
||||
def test_no_config_defaults(self):
|
||||
# No reasoning_config → thinking on, server picks depth; no effort.
|
||||
p = get_provider_profile("kimi")
|
||||
eb, tl = p.build_api_kwargs_extras(reasoning_config=None)
|
||||
assert eb["thinking"] == {"type": "enabled"}
|
||||
assert "reasoning_effort" not in tl
|
||||
|
||||
|
||||
class TestOpenRouterProfile:
|
||||
def test_extra_body_with_prefs(self):
|
||||
p = get_provider_profile("openrouter")
|
||||
body = p.build_extra_body(provider_preferences={"allow": ["anthropic"]})
|
||||
assert body["provider"] == {"allow": ["anthropic"]}
|
||||
|
||||
def test_extra_body_session_id(self):
|
||||
p = get_provider_profile("openrouter")
|
||||
body = p.build_extra_body(session_id="test-session-123")
|
||||
assert body["session_id"] == "test-session-123"
|
||||
|
||||
def test_extra_body_no_prefs(self):
|
||||
p = get_provider_profile("openrouter")
|
||||
body = p.build_extra_body()
|
||||
assert body == {}
|
||||
|
||||
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
|
||||
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
|
||||
p = get_provider_profile("openrouter")
|
||||
body = p.build_extra_body(
|
||||
model="openrouter/pareto-code",
|
||||
openrouter_min_coding_score=0.65,
|
||||
)
|
||||
assert body["plugins"] == [
|
||||
{"id": "pareto-router", "min_coding_score": 0.65}
|
||||
]
|
||||
|
||||
def test_pareto_score_ignored_for_other_models(self):
|
||||
"""Score has no effect on any other model — plugins block must not appear."""
|
||||
p = get_provider_profile("openrouter")
|
||||
body = p.build_extra_body(
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
openrouter_min_coding_score=0.65,
|
||||
)
|
||||
assert "plugins" not in body
|
||||
|
||||
def test_pareto_score_unset_omits_plugins(self):
|
||||
"""Empty/None score → no plugins block (router uses its omission default)."""
|
||||
p = get_provider_profile("openrouter")
|
||||
for unset in (None, ""):
|
||||
body = p.build_extra_body(
|
||||
model="openrouter/pareto-code",
|
||||
openrouter_min_coding_score=unset,
|
||||
)
|
||||
assert "plugins" not in body, f"unset={unset!r}"
|
||||
|
||||
def test_pareto_score_out_of_range_dropped(self):
|
||||
"""Invalid scores are silently dropped — never forwarded to OR."""
|
||||
p = get_provider_profile("openrouter")
|
||||
for bad in (1.5, -0.1, "not-a-number"):
|
||||
body = p.build_extra_body(
|
||||
model="openrouter/pareto-code",
|
||||
openrouter_min_coding_score=bad,
|
||||
)
|
||||
assert "plugins" not in body, f"bad={bad!r}"
|
||||
|
||||
def test_reasoning_full_config(self):
|
||||
p = get_provider_profile("openrouter")
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": True, "effort": "high"}
|
||||
|
||||
def test_reasoning_disabled_still_passes(self):
|
||||
"""OpenRouter passes disabled reasoning through (unlike Nous)."""
|
||||
p = get_provider_profile("openrouter")
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": False}
|
||||
|
||||
def test_reasoning_disable_omitted_for_mandatory_anthropic(self):
|
||||
"""Reasoning-mandatory Anthropic models (4.6+/fable) reject any disable
|
||||
form: OpenRouter translates ``reasoning: {enabled: false}`` into
|
||||
Anthropic's ``thinking: {type: disabled}``, which 400s. The profile must
|
||||
omit ``reasoning`` so the model falls back to adaptive thinking instead.
|
||||
"""
|
||||
p = get_provider_profile("openrouter")
|
||||
for model in (
|
||||
"anthropic/claude-fable-5", # new named model
|
||||
"anthropic/claude-some-future-7", # unknown → default mandatory
|
||||
"anthropic/claude-opus-4.8",
|
||||
"anthropic/claude-opus-4.6",
|
||||
):
|
||||
for cfg in ({"enabled": False}, {"effort": "none"}):
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config=cfg,
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert "reasoning" not in eb, (model, cfg, eb)
|
||||
|
||||
def test_reasoning_disable_kept_for_legacy_anthropic(self):
|
||||
"""Older Anthropic models still accept an explicit disable form, so the
|
||||
profile must keep forwarding it."""
|
||||
p = get_provider_profile("openrouter")
|
||||
for model in (
|
||||
"anthropic/claude-3.7-sonnet",
|
||||
"anthropic/claude-opus-4.5",
|
||||
"anthropic/claude-sonnet-4.5",
|
||||
):
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": False}, (model, eb)
|
||||
|
||||
def test_reasoning_disable_kept_for_non_anthropic(self):
|
||||
"""Non-Anthropic models (DeepSeek, Qwen, …) disable reasoning fine; the
|
||||
Anthropic-mandatory guard must not touch them."""
|
||||
p = get_provider_profile("openrouter")
|
||||
for model in ("deepseek/deepseek-chat", "qwen/qwen3-max", "openai/gpt-5.4"):
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": False}, (model, eb)
|
||||
|
||||
def test_reasoning_omitted_for_mandatory_anthropic_even_when_enabled(self):
|
||||
"""Reasoning-mandatory Anthropic models (4.6+/fable) use adaptive
|
||||
thinking — OpenRouter ignores reasoning.effort for them, and sending any
|
||||
reasoning field makes OpenRouter emit thinking.type.disabled on
|
||||
tool-continuation turns (whose assistant tool_calls carry no thinking
|
||||
block), 400ing every turn after the first tool call. The profile must
|
||||
omit reasoning entirely so the model defaults to adaptive.
|
||||
"""
|
||||
p = get_provider_profile("openrouter")
|
||||
for cfg in (
|
||||
{"enabled": True, "effort": "medium"},
|
||||
{"enabled": True, "effort": "xhigh"},
|
||||
{"effort": "high"},
|
||||
{"enabled": True},
|
||||
):
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config=cfg,
|
||||
supports_reasoning=True,
|
||||
model="anthropic/claude-fable-5",
|
||||
)
|
||||
assert "reasoning" not in eb, (cfg, eb)
|
||||
|
||||
def test_default_reasoning(self):
|
||||
p = get_provider_profile("openrouter")
|
||||
eb, _ = p.build_api_kwargs_extras(supports_reasoning=True)
|
||||
assert eb["reasoning"] == {"enabled": True, "effort": "medium"}
|
||||
|
||||
def test_grok_session_id_sets_cache_affinity_header(self):
|
||||
"""OpenRouter + Grok model + session_id => x-grok-conv-id header."""
|
||||
p = get_provider_profile("openrouter")
|
||||
_, tl = p.build_api_kwargs_extras(
|
||||
model="x-ai/grok-4",
|
||||
session_id="sess-abc123",
|
||||
)
|
||||
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-abc123"
|
||||
|
||||
def test_grok_xai_prefix_also_supported(self):
|
||||
"""xai/ prefix (without dash) should also get the header."""
|
||||
p = get_provider_profile("openrouter")
|
||||
_, tl = p.build_api_kwargs_extras(
|
||||
model="xai/grok-3",
|
||||
session_id="sess-xyz",
|
||||
)
|
||||
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-xyz"
|
||||
|
||||
def test_non_grok_model_no_affinity_header(self):
|
||||
"""OpenRouter + non-Grok model => no x-grok-conv-id header."""
|
||||
p = get_provider_profile("openrouter")
|
||||
_, tl = p.build_api_kwargs_extras(
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
session_id="sess-abc123",
|
||||
)
|
||||
assert "extra_headers" not in tl
|
||||
assert "x-grok-conv-id" not in tl
|
||||
|
||||
def test_grok_without_session_id_no_header(self):
|
||||
"""Grok model but no session_id => no header (nothing to pin)."""
|
||||
p = get_provider_profile("openrouter")
|
||||
_, tl = p.build_api_kwargs_extras(model="x-ai/grok-4")
|
||||
assert "extra_headers" not in tl
|
||||
|
||||
def test_grok_reasoning_and_header_together(self):
|
||||
"""Reasoning extra_body and Grok header should coexist."""
|
||||
p = get_provider_profile("openrouter")
|
||||
eb, tl = p.build_api_kwargs_extras(
|
||||
model="x-ai/grok-4",
|
||||
session_id="sess-123",
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": True, "effort": "high"}
|
||||
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-123"
|
||||
|
||||
# --- reasoning-mandatory Anthropic effort → top-level verbosity (#43432) ---
|
||||
#
|
||||
# These models (Claude 4.6+ / fable / mythos-class) ignore
|
||||
# ``reasoning.effort`` and use adaptive thinking. OpenRouter honors the
|
||||
# requested effort on the top-level ``verbosity`` field instead (maps to
|
||||
# Anthropic ``output_config.effort``). The profile must route the existing
|
||||
# ``reasoning_config["effort"]`` there while still NEVER emitting a
|
||||
# ``reasoning`` field (which would 400 — see #42991). Gate every fixture on
|
||||
# the real predicate so this stays a behavior contract, not a name snapshot.
|
||||
|
||||
@staticmethod
|
||||
def _is_mandatory(model):
|
||||
import inspect
|
||||
p = get_provider_profile("openrouter")
|
||||
mod = inspect.getmodule(type(p))
|
||||
return mod._anthropic_reasoning_is_mandatory(model)
|
||||
|
||||
def test_mandatory_anthropic_effort_routes_to_verbosity(self):
|
||||
"""effort set + reasoning enabled → top-level verbosity == effort,
|
||||
and NO reasoning field in extra_body.
|
||||
|
||||
Covers the full real config range produced by
|
||||
``hermes_constants.parse_reasoning_effort`` —
|
||||
``VALID_REASONING_EFFORTS = (minimal, low, medium, high, xhigh)``.
|
||||
"""
|
||||
p = get_provider_profile("openrouter")
|
||||
model = "anthropic/claude-fable-5"
|
||||
assert self._is_mandatory(model) # fixture really is mandatory
|
||||
for effort in ("minimal", "low", "medium", "high", "xhigh"):
|
||||
eb, tl = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert tl["verbosity"] == effort, (effort, tl)
|
||||
assert "reasoning" not in eb, (effort, eb)
|
||||
|
||||
def test_mandatory_anthropic_effort_without_enabled_key_routes(self):
|
||||
"""effort present without an explicit ``enabled`` key still routes to
|
||||
verbosity (enabled defaults to True)."""
|
||||
p = get_provider_profile("openrouter")
|
||||
eb, tl = p.build_api_kwargs_extras(
|
||||
reasoning_config={"effort": "xhigh"},
|
||||
supports_reasoning=True,
|
||||
model="anthropic/claude-fable-5",
|
||||
)
|
||||
assert tl["verbosity"] == "xhigh"
|
||||
assert "reasoning" not in eb
|
||||
|
||||
def test_mandatory_anthropic_verbosity_is_value_agnostic_passthrough(self):
|
||||
"""The mapping passes the effort value through verbatim — it must NOT
|
||||
clamp or whitelist. ``xhigh`` is a real config value; ``max`` is not
|
||||
producible by ``parse_reasoning_effort`` today but OpenRouter accepts it
|
||||
for Claude (live-proven in #43432), so a forward value must survive
|
||||
rather than be silently dropped. The OpenAI SDK type only literals
|
||||
``low|medium|high`` but it's a TypedDict (no runtime validation), so the
|
||||
extended scale reaches the wire untouched."""
|
||||
p = get_provider_profile("openrouter")
|
||||
for effort in ("xhigh", "max"):
|
||||
_, tl = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
supports_reasoning=True,
|
||||
model="anthropic/claude-fable-5",
|
||||
)
|
||||
assert tl["verbosity"] == effort
|
||||
|
||||
def test_mandatory_anthropic_no_verbosity_when_effort_absent(self):
|
||||
"""No effort / none / disabled → no verbosity emitted, so the model
|
||||
keeps its own adaptive default. Still no reasoning field."""
|
||||
p = get_provider_profile("openrouter")
|
||||
model = "anthropic/claude-fable-5"
|
||||
for cfg in (
|
||||
None,
|
||||
{},
|
||||
{"enabled": True},
|
||||
{"effort": "none"},
|
||||
{"enabled": True, "effort": "none"},
|
||||
{"enabled": False, "effort": "high"}, # explicitly disabled wins
|
||||
):
|
||||
eb, tl = p.build_api_kwargs_extras(
|
||||
reasoning_config=cfg,
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert "verbosity" not in tl, (cfg, tl)
|
||||
assert "reasoning" not in eb, (cfg, eb)
|
||||
|
||||
def test_non_mandatory_reasoning_model_unchanged_no_verbosity(self):
|
||||
"""Non-mandatory reasoning models (DeepSeek, Qwen, GPT) keep getting
|
||||
``reasoning`` in extra_body and never get a ``verbosity`` field — the
|
||||
new path must not touch them."""
|
||||
p = get_provider_profile("openrouter")
|
||||
for model in ("deepseek/deepseek-chat", "qwen/qwen3-max", "openai/gpt-5.4"):
|
||||
assert not self._is_mandatory(model) # fixture really is non-mandatory
|
||||
eb, tl = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": True, "effort": "high"}, (model, eb)
|
||||
assert "verbosity" not in tl, (model, tl)
|
||||
|
||||
def test_mandatory_anthropic_verbosity_coexists_with_grok_header(self):
|
||||
"""A reasoning-mandatory Anthropic model is never a Grok model, but the
|
||||
top-level dict must remain a single merged dict — verify the verbosity
|
||||
path doesn't clobber the extra_headers slot used by Grok affinity."""
|
||||
p = get_provider_profile("openrouter")
|
||||
# mandatory anthropic + effort → verbosity, no extra_headers
|
||||
_, tl = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
supports_reasoning=True,
|
||||
model="anthropic/claude-fable-5",
|
||||
)
|
||||
assert tl == {"verbosity": "high"}
|
||||
|
||||
|
||||
class TestNousProfile:
|
||||
def test_tags(self):
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
p = get_provider_profile("nous")
|
||||
body = p.build_extra_body()
|
||||
assert body["tags"] == nous_portal_tags()
|
||||
|
||||
def test_auth_type(self):
|
||||
p = get_provider_profile("nous")
|
||||
assert p.auth_type == "oauth_device_code"
|
||||
|
||||
def test_reasoning_enabled(self):
|
||||
p = get_provider_profile("nous")
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert eb["reasoning"] == {"enabled": True, "effort": "medium"}
|
||||
|
||||
def test_reasoning_omitted_when_disabled(self):
|
||||
p = get_provider_profile("nous")
|
||||
eb, _ = p.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert "reasoning" not in eb
|
||||
|
||||
|
||||
class TestQwenProfile:
|
||||
def test_max_tokens(self):
|
||||
p = get_provider_profile("qwen-oauth")
|
||||
assert p.default_max_tokens == 65536
|
||||
|
||||
def test_auth_type(self):
|
||||
p = get_provider_profile("qwen-oauth")
|
||||
assert p.auth_type == "oauth_external"
|
||||
|
||||
def test_extra_body_vl(self):
|
||||
p = get_provider_profile("qwen-oauth")
|
||||
body = p.build_extra_body()
|
||||
assert body["vl_high_resolution_images"] is True
|
||||
|
||||
def test_prepare_messages_normalizes_content(self):
|
||||
p = get_provider_profile("qwen-oauth")
|
||||
msgs = [
|
||||
{"role": "system", "content": "Be helpful"},
|
||||
{"role": "user", "content": "hello"},
|
||||
]
|
||||
result = p.prepare_messages(msgs)
|
||||
# System message: content normalized to list, cache_control on last part
|
||||
assert isinstance(result[0]["content"], list)
|
||||
assert result[0]["content"][-1].get("cache_control") == {"type": "ephemeral"}
|
||||
assert result[0]["content"][-1]["text"] == "Be helpful"
|
||||
# User message: content normalized to list
|
||||
assert isinstance(result[1]["content"], list)
|
||||
assert result[1]["content"][0]["text"] == "hello"
|
||||
|
||||
def test_metadata_top_level(self):
|
||||
p = get_provider_profile("qwen-oauth")
|
||||
meta = {"sessionId": "s123", "promptId": "p456"}
|
||||
eb, tl = p.build_api_kwargs_extras(qwen_session_metadata=meta)
|
||||
assert tl["metadata"] == meta
|
||||
assert "metadata" not in eb
|
||||
|
||||
|
||||
class TestBaseProfile:
|
||||
def test_prepare_messages_passthrough(self):
|
||||
p = ProviderProfile(name="test")
|
||||
msgs = [{"role": "user", "content": "hi"}]
|
||||
assert p.prepare_messages(msgs) is msgs
|
||||
|
||||
def test_build_extra_body_empty(self):
|
||||
p = ProviderProfile(name="test")
|
||||
assert p.build_extra_body() == {}
|
||||
|
||||
def test_build_api_kwargs_extras_empty(self):
|
||||
p = ProviderProfile(name="test")
|
||||
eb, tl = p.build_api_kwargs_extras()
|
||||
assert eb == {}
|
||||
assert tl == {}
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Parity tests: pin the exact current transport behavior per provider.
|
||||
|
||||
These tests document the flag-based contract between run_agent.py and
|
||||
ChatCompletionsTransport.build_kwargs(). When the next PR wires profiles
|
||||
to replace flags, every assertion here must still pass — any failure is
|
||||
a behavioral regression.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
from providers import get_provider_profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport():
|
||||
return ChatCompletionsTransport()
|
||||
|
||||
|
||||
def _simple_messages():
|
||||
return [{"role": "user", "content": "hello"}]
|
||||
|
||||
|
||||
def _max_tokens_fn(n):
|
||||
return {"max_completion_tokens": n}
|
||||
|
||||
|
||||
class TestNvidiaParity:
|
||||
"""NVIDIA NIM: default max_tokens=16384."""
|
||||
|
||||
def test_default_max_tokens(self, transport):
|
||||
"""NVIDIA default max_tokens=16384 comes from profile, not legacy is_nvidia_nim flag."""
|
||||
from providers import get_provider_profile
|
||||
|
||||
profile = get_provider_profile("nvidia")
|
||||
kw = transport.build_kwargs(
|
||||
model="nvidia/llama-3.1-nemotron-70b-instruct",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
provider_profile=profile,
|
||||
)
|
||||
assert kw["max_completion_tokens"] == 16384
|
||||
|
||||
def test_user_max_tokens_overrides(self, transport):
|
||||
from providers import get_provider_profile
|
||||
|
||||
profile = get_provider_profile("nvidia")
|
||||
kw = transport.build_kwargs(
|
||||
model="nvidia/llama-3.1-nemotron-70b-instruct",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
provider_profile=profile,
|
||||
)
|
||||
assert kw["max_completion_tokens"] == 4096 # user overrides default
|
||||
|
||||
|
||||
class TestKimiParity:
|
||||
"""Kimi: OMIT temperature, max_tokens=32000, thinking + reasoning_effort."""
|
||||
|
||||
def test_temperature_omitted(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
omit_temperature=True,
|
||||
)
|
||||
assert "temperature" not in kw
|
||||
|
||||
def test_default_max_tokens(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
assert kw["max_completion_tokens"] == 32000
|
||||
|
||||
def test_thinking_enabled(self, transport):
|
||||
# xor contract (fix ce4e74b3): an explicit recognized effort sends
|
||||
# reasoning_effort ONLY — never paired with extra_body.thinking.
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning_effort") == "high"
|
||||
assert "thinking" not in kw.get("extra_body", {})
|
||||
|
||||
def test_thinking_enabled_without_effort(self, transport):
|
||||
# enabled but no effort → fall back to the thinking toggle, no effort.
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
reasoning_config={"enabled": True},
|
||||
)
|
||||
assert kw["extra_body"]["thinking"] == {"type": "enabled"}
|
||||
assert "reasoning_effort" not in kw
|
||||
|
||||
def test_thinking_disabled(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
reasoning_config={"enabled": False},
|
||||
)
|
||||
assert kw["extra_body"]["thinking"] == {"type": "disabled"}
|
||||
assert "reasoning_effort" not in kw
|
||||
|
||||
def test_reasoning_effort_top_level(self, transport):
|
||||
"""Kimi reasoning_effort is a TOP-LEVEL api_kwargs key, NOT in extra_body."""
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning_effort") == "high"
|
||||
assert "reasoning_effort" not in kw.get("extra_body", {})
|
||||
|
||||
def test_reasoning_effort_default_no_effort(self, transport):
|
||||
# xor contract: enabled with no effort falls back to thinking-enabled
|
||||
# and emits NO top-level reasoning_effort (previously defaulted to
|
||||
# "medium" alongside thinking — the pairing this fix removes).
|
||||
kw = transport.build_kwargs(
|
||||
model="kimi-k2",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("kimi-coding"),
|
||||
reasoning_config={"enabled": True},
|
||||
)
|
||||
assert "reasoning_effort" not in kw
|
||||
assert kw["extra_body"]["thinking"] == {"type": "enabled"}
|
||||
|
||||
|
||||
class TestOpenRouterParity:
|
||||
"""OpenRouter: provider preferences, reasoning in extra_body."""
|
||||
|
||||
def test_provider_preferences(self, transport):
|
||||
prefs = {"allow": ["anthropic"], "sort": "price"}
|
||||
kw = transport.build_kwargs(
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
provider_preferences=prefs,
|
||||
)
|
||||
assert kw["extra_body"]["provider"] == prefs
|
||||
|
||||
def test_reasoning_passes_full_config(self, transport):
|
||||
"""OpenRouter passes the FULL reasoning_config dict, not just effort."""
|
||||
rc = {"enabled": True, "effort": "high"}
|
||||
kw = transport.build_kwargs(
|
||||
model="deepseek/deepseek-chat",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
supports_reasoning=True,
|
||||
reasoning_config=rc,
|
||||
)
|
||||
assert kw["extra_body"]["reasoning"] == rc
|
||||
|
||||
def test_reasoning_omitted_for_mandatory_anthropic(self, transport):
|
||||
"""Adaptive-thinking Anthropic models (4.6+/fable) get NO reasoning
|
||||
field — sending one makes OpenRouter emit thinking.type.disabled on
|
||||
tool-replay turns, which the model 400s on."""
|
||||
kw = transport.build_kwargs(
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw.get("extra_body", {})
|
||||
|
||||
def test_default_reasoning_when_no_config(self, transport):
|
||||
"""When supports_reasoning=True but no config, adds default."""
|
||||
kw = transport.build_kwargs(
|
||||
model="deepseek/deepseek-chat",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("openrouter"),
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert kw["extra_body"]["reasoning"] == {"enabled": True, "effort": "medium"}
|
||||
|
||||
|
||||
class TestNousParity:
|
||||
"""Nous: product tags, reasoning, omit when disabled."""
|
||||
|
||||
def test_tags(self, transport):
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
kw = transport.build_kwargs(
|
||||
model="hermes-3-llama-3.1-405b",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("nous"),
|
||||
)
|
||||
assert kw["extra_body"]["tags"] == nous_portal_tags()
|
||||
|
||||
def test_reasoning_omitted_when_disabled(self, transport):
|
||||
"""Nous special case: reasoning omitted entirely when disabled."""
|
||||
kw = transport.build_kwargs(
|
||||
model="hermes-3-llama-3.1-405b",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("nous"),
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": False},
|
||||
)
|
||||
assert "reasoning" not in kw.get("extra_body", {})
|
||||
|
||||
def test_reasoning_enabled(self, transport):
|
||||
rc = {"enabled": True, "effort": "high"}
|
||||
kw = transport.build_kwargs(
|
||||
model="hermes-3-llama-3.1-405b",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("nous"),
|
||||
supports_reasoning=True,
|
||||
reasoning_config=rc,
|
||||
)
|
||||
assert kw["extra_body"]["reasoning"] == rc
|
||||
|
||||
|
||||
class TestQwenParity:
|
||||
"""Qwen: max_tokens=65536, vl_high_resolution, metadata top-level."""
|
||||
|
||||
def test_default_max_tokens(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="qwen3.5-plus",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("qwen-oauth"),
|
||||
max_tokens_param_fn=_max_tokens_fn,
|
||||
)
|
||||
assert kw["max_completion_tokens"] == 65536
|
||||
|
||||
def test_vl_high_resolution(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="qwen3.5-plus",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("qwen-oauth"),
|
||||
)
|
||||
assert kw["extra_body"]["vl_high_resolution_images"] is True
|
||||
|
||||
def test_metadata_top_level(self, transport):
|
||||
"""Qwen metadata goes to top-level api_kwargs, NOT extra_body."""
|
||||
meta = {"sessionId": "s123", "promptId": "p456"}
|
||||
kw = transport.build_kwargs(
|
||||
model="qwen3.5-plus",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("qwen-oauth"),
|
||||
qwen_session_metadata=meta,
|
||||
)
|
||||
assert kw["metadata"] == meta
|
||||
assert "metadata" not in kw.get("extra_body", {})
|
||||
|
||||
|
||||
class TestCustomOllamaParity:
|
||||
"""Custom/Ollama: num_ctx, thinking controls — now tested via profile."""
|
||||
|
||||
def test_ollama_num_ctx(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="llama3.1",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("custom"),
|
||||
ollama_num_ctx=131072,
|
||||
)
|
||||
assert kw["extra_body"]["options"]["num_ctx"] == 131072
|
||||
|
||||
def test_think_false_when_disabled(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="qwen3:72b",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("custom"),
|
||||
reasoning_config={"enabled": False, "effort": "none"},
|
||||
)
|
||||
assert kw["extra_body"]["think"] is False
|
||||
Reference in New Issue
Block a user