Hermes-agent

This commit is contained in:
Zakaria
2026-06-14 14:30:48 -04:00
commit dac4b88b94
5058 changed files with 1884848 additions and 0 deletions
View File
@@ -0,0 +1,300 @@
"""Behavior-parity check for the image-gen FAL plugin migration (#26241).
Spawns one subprocess per (version, scenario) cell — pinned to either
``origin/main`` (legacy in-tree FAL fall-through + ``configured == "fal"``
skip in ``_dispatch_to_plugin_provider``) or this PR's worktree (FAL is
itself a plugin and the dispatcher routes every set provider through
the registry). Each subprocess clears all FAL-related env vars + writes
a ``config.yaml``, then asks the dispatcher how it would route an
``image_generate`` call. The emitted shape tuple is
``{dispatch_kind, provider_name, model}``:
* ``dispatch_kind`` ∈ ``{"legacy_fal", "plugin", "error", None}`` —
whether the call would go straight to the in-tree pipeline,
through ``_dispatch_to_plugin_provider``, raise an explicit
provider-not-registered error, or fall through silently.
* ``provider_name`` — when ``dispatch_kind == "plugin"``, the
resolved provider name. ``None`` otherwise.
* ``model`` — the resolved FAL model id when applicable.
The parent process diffs the shapes per scenario. A diff means the
migration introduced an observable behaviour change vs origin/main —
likely a real regression for users on the existing config keys.
Run from the PR worktree:
python tests/plugins/image_gen/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
# Pin one path to current main, one to the PR worktree.
# ``REPO_ROOT`` is ``.../.worktrees/<name>``; the main checkout lives
# two levels up. When running directly from a regular clone (no
# worktree), ``MAIN_DIR`` falls back to a sibling ``hermes-agent-main``
# checkout if one exists.
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "image_generation_tool.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "image_generation_tool.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "image_generation_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear FAL-related env so dispatch decisions are config-driven.
for k in (
"FAL_KEY", "FAL_QUEUE_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
"FAL_IMAGE_MODEL",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
import tools.image_generation_tool as image_tool
dispatch_kind = None
provider_name = None
model = None
error_text = None
try:
raw = image_tool._dispatch_to_plugin_provider("ping", "landscape")
if raw is None:
dispatch_kind = "legacy_fal"
else:
parsed = json.loads(raw) if isinstance(raw, str) else raw
if isinstance(parsed, dict):
if parsed.get("error_type") == "provider_not_registered":
dispatch_kind = "error"
error_text = parsed.get("error")
else:
dispatch_kind = "plugin"
provider_name = parsed.get("provider")
model = parsed.get("model")
else:
dispatch_kind = "unknown_payload"
if model is None:
# _resolve_fal_model still returns the active FAL model id even
# when dispatch goes to a non-FAL plugin — used for the diff
# only when applicable.
try:
model_id, _meta = image_tool._resolve_fal_model()
if dispatch_kind == "legacy_fal":
model = model_id
except Exception:
pass
except Exception as exc:
dispatch_kind = "exception"
error_text = repr(exc)
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"model": model,
"error_present": error_text is not None,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
# (label, config.yaml body, extra env vars)
("no-config-no-env", "", {}),
(
"explicit-fal-no-creds",
"image_gen:\n provider: fal\n",
{},
),
(
"explicit-fal-with-creds",
"image_gen:\n provider: fal\n",
{"FAL_KEY": "test-key"},
),
(
"explicit-fal-with-model",
"image_gen:\n provider: fal\n model: fal-ai/flux-2-pro\n",
{"FAL_KEY": "test-key"},
),
(
"explicit-typo-provider",
"image_gen:\n provider: not-a-real-backend\n",
{"FAL_KEY": "test-key"},
),
(
"managed-gateway-only",
"",
{
"TOOL_GATEWAY_DOMAIN": "nousresearch.com",
"TOOL_GATEWAY_USER_TOKEN": "nous-token",
},
),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
"""Reduce to the parts that matter for user-visible parity.
On origin/main, ``explicit-fal-*`` scenarios short-circuit to
``legacy_fal`` because of the ``configured == "fal"`` skip. On the
PR, those same scenarios route through the plugin and emit
``dispatch_kind == "plugin"`` with ``provider_name == "fal"``.
Both shapes are functionally equivalent — the plugin's ``generate()``
re-enters the same in-tree pipeline via ``_it`` indirection — but
we want the diff to be visible so reviewers can sign off on the
intentional behaviour delta.
"""
return {
"dispatch_kind": shape.get("dispatch_kind"),
"provider_name": shape.get("provider_name"),
"model": shape.get("model"),
"error_present": shape.get("error_present"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "explicit-fal-*" returns legacy_fal; on PR, plugin
# dispatch. That's the only acceptable diff — flag everything
# else as a regression.
legacy_to_plugin_fal = (
main_reduced.get("dispatch_kind") == "legacy_fal"
and pr_reduced.get("dispatch_kind") == "plugin"
and pr_reduced.get("provider_name") == "fal"
)
if legacy_to_plugin_fal:
print(f" [DIFF] {label}: legacy_fal → plugin (fal) — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"legacy_fal → plugin dispatch for explicit FAL paths."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,225 @@
#!/usr/bin/env python3
"""Tests for the FAL.ai image generation plugin.
The plugin is a thin registration adapter — actual FAL pipeline logic
lives in ``tools.image_generation_tool`` and is exercised by
``tests/tools/test_image_generation.py``. These tests focus on:
* the ``ImageGenProvider`` ABC surface (name, models, schema)
* call-time indirection (``_it`` resolution at ``generate()`` time so
``monkeypatch.setattr(image_tool, ...)`` keeps working)
* response shape stamping (provider/prompt/aspect_ratio/model)
"""
from __future__ import annotations
import json
from unittest.mock import MagicMock
# ---------------------------------------------------------------------------
# Provider surface
# ---------------------------------------------------------------------------
class TestFalImageGenProviderSurface:
def test_name(self):
from plugins.image_gen.fal import FalImageGenProvider
assert FalImageGenProvider().name == "fal"
def test_display_name(self):
from plugins.image_gen.fal import FalImageGenProvider
assert FalImageGenProvider().display_name == "FAL.ai"
def test_default_model_matches_legacy(self):
from plugins.image_gen.fal import FalImageGenProvider
from tools.image_generation_tool import DEFAULT_MODEL
assert FalImageGenProvider().default_model() == DEFAULT_MODEL
def test_list_models_uses_legacy_catalog(self):
from plugins.image_gen.fal import FalImageGenProvider
from tools.image_generation_tool import FAL_MODELS
provider = FalImageGenProvider()
models = provider.list_models()
ids = {m["id"] for m in models}
# Whatever FAL_MODELS ships, the provider mirrors verbatim.
assert ids == set(FAL_MODELS.keys())
# Spot-check the expected first-class fields are present.
for entry in models:
for field in ("id", "display", "speed", "strengths", "price"):
assert field in entry
def test_setup_schema_advertises_fal_key(self):
from plugins.image_gen.fal import FalImageGenProvider
schema = FalImageGenProvider().get_setup_schema()
assert schema["name"] == "FAL.ai"
assert schema["badge"] == "paid"
env_keys = {entry["key"] for entry in schema.get("env_vars", [])}
assert "FAL_KEY" in env_keys
class TestFalImageGenProviderAvailability:
def test_is_available_when_legacy_check_passes(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: True)
assert FalImageGenProvider().is_available() is True
def test_is_available_false_when_legacy_check_fails(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: False)
assert FalImageGenProvider().is_available() is False
def test_is_available_handles_legacy_exception(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
def _boom():
raise RuntimeError("config broke")
monkeypatch.setattr(image_tool, "check_fal_api_key", _boom)
# Picker must not propagate exceptions — show as "not available".
assert FalImageGenProvider().is_available() is False
# ---------------------------------------------------------------------------
# generate() — call-time indirection
# ---------------------------------------------------------------------------
class TestFalImageGenProviderGenerate:
def test_generate_delegates_to_legacy_image_generate_tool(self, monkeypatch):
"""Plugin must look up ``image_generate_tool`` at call time so
``monkeypatch.setattr(image_tool, "image_generate_tool", ...)``
takes effect."""
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
captured = {}
def fake_image_generate_tool(prompt, aspect_ratio, **kwargs):
captured["prompt"] = prompt
captured["aspect_ratio"] = aspect_ratio
captured["kwargs"] = kwargs
return json.dumps({"success": True, "image": "https://fake/image.png"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake_image_generate_tool)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
result = FalImageGenProvider().generate(
"a serene mountain landscape",
aspect_ratio="square",
seed=42,
)
assert captured["prompt"] == "a serene mountain landscape"
assert captured["aspect_ratio"] == "square"
assert captured["kwargs"] == {"seed": 42}
assert result["success"] is True
assert result["image"] == "https://fake/image.png"
# Stamped fields for the unified response shape
assert result["provider"] == "fal"
assert result["prompt"] == "a serene mountain landscape"
assert result["aspect_ratio"] == "square"
assert result["model"] == "fal-ai/flux-2/klein/9b"
def test_generate_invalid_aspect_ratio_is_coerced(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
seen_aspect = {}
def fake(prompt, aspect_ratio, **kwargs):
seen_aspect["v"] = aspect_ratio
return json.dumps({"success": True, "image": "x"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
FalImageGenProvider().generate("p", aspect_ratio="not-a-real-ratio")
# ``resolve_aspect_ratio`` clamps to landscape.
assert seen_aspect["v"] == "landscape"
def test_generate_passthrough_drops_none_kwargs(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
seen = {}
def fake(prompt, aspect_ratio, **kwargs):
seen.update(kwargs)
return json.dumps({"success": True, "image": "x"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
FalImageGenProvider().generate(
"p",
aspect_ratio="landscape",
seed=None,
num_images=2,
guidance_scale=None,
)
# ``None`` values must not be forwarded — they'd override the
# model's defaults inside the legacy payload builder.
assert "seed" not in seen
assert "guidance_scale" not in seen
assert seen.get("num_images") == 2
def test_generate_catches_exception_from_legacy(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
def boom(*args, **kwargs):
raise RuntimeError("FAL endpoint exploded")
monkeypatch.setattr(image_tool, "image_generate_tool", boom)
result = FalImageGenProvider().generate("p")
assert result["success"] is False
assert "FAL image generation failed" in result["error"]
assert result["error_type"] == "RuntimeError"
assert result["provider"] == "fal"
def test_generate_invalid_json_response(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "image_generate_tool", lambda **kw: "not-json")
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
result = FalImageGenProvider().generate("p")
assert result["success"] is False
assert "Invalid JSON" in result["error"]
assert result["provider"] == "fal"
# ---------------------------------------------------------------------------
# Registry wiring
# ---------------------------------------------------------------------------
class TestFalImageGenPluginRegistration:
def test_register_wires_provider_into_registry(self):
from plugins.image_gen.fal import FalImageGenProvider, register
ctx = MagicMock()
register(ctx)
ctx.register_image_gen_provider.assert_called_once()
(registered,), _ = ctx.register_image_gen_provider.call_args
assert isinstance(registered, FalImageGenProvider)
@@ -0,0 +1,625 @@
#!/usr/bin/env python3
"""Tests for Krea image generation provider."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _fake_api_key(monkeypatch):
"""Ensure KREA_API_KEY is set for all tests."""
monkeypatch.setenv("KREA_API_KEY", "test-key-12345")
def _completed_job(url: str = "https://krea.cdn/img.png") -> dict:
return {
"job_id": "00000000-0000-0000-0000-000000000abc",
"status": "completed",
"created_at": "2026-05-27T00:00:00Z",
"completed_at": "2026-05-27T00:00:30Z",
"result": {"urls": [url]},
}
def _submit_response(job_id: str = "00000000-0000-0000-0000-000000000abc"):
resp = MagicMock()
resp.status_code = 200
resp.raise_for_status = MagicMock()
resp.json.return_value = {
"job_id": job_id,
"status": "queued",
"created_at": "2026-05-27T00:00:00Z",
"completed_at": None,
"result": None,
}
return resp
def _poll_response(body: dict):
resp = MagicMock()
resp.status_code = 200
resp.raise_for_status = MagicMock()
resp.json.return_value = body
return resp
# ---------------------------------------------------------------------------
# Provider class tests
# ---------------------------------------------------------------------------
class TestKreaImageGenProvider:
def test_name(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().name == "krea"
def test_display_name(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().display_name == "Krea"
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("KREA_API_KEY", "sk-test")
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().is_available() is True
def test_is_available_without_key(self, monkeypatch):
monkeypatch.delenv("KREA_API_KEY", raising=False)
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().is_available() is False
def test_list_models(self):
from plugins.image_gen.krea import KreaImageGenProvider
models = KreaImageGenProvider().list_models()
ids = {m["id"] for m in models}
assert {"krea-2-medium", "krea-2-large"} <= ids
# Each entry carries the picker fields the registry expects.
for m in models:
assert m["display"]
assert m["speed"]
assert m["strengths"]
assert m["price"]
def test_default_model_is_medium(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().default_model() == "krea-2-medium"
def test_get_setup_schema(self):
from plugins.image_gen.krea import KreaImageGenProvider
schema = KreaImageGenProvider().get_setup_schema()
assert schema["name"] == "Krea"
assert schema["badge"] == "paid"
env_vars = schema["env_vars"]
assert len(env_vars) == 1
assert env_vars[0]["key"] == "KREA_API_KEY"
assert "krea.ai" in env_vars[0]["url"]
# ---------------------------------------------------------------------------
# Model resolution
# ---------------------------------------------------------------------------
class TestModelResolution:
def test_default(self):
from plugins.image_gen.krea import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "krea-2-medium"
assert meta["path"] == "medium"
def test_env_override_large(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
from plugins.image_gen.krea import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "krea-2-large"
assert meta["path"] == "large"
def test_env_override_unknown_falls_back_to_default(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-xxl-fake")
from plugins.image_gen.krea import _resolve_model
model_id, _ = _resolve_model()
assert model_id == "krea-2-medium"
def test_creativity_default(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity(None) == "medium"
def test_creativity_valid(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity("HIGH") == "high"
assert _resolve_creativity(" raw ") == "raw"
def test_creativity_invalid(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity("ultra") == "medium"
# ---------------------------------------------------------------------------
# Generate — main flow
# ---------------------------------------------------------------------------
class TestGenerate:
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("KREA_API_KEY", raising=False)
from plugins.image_gen.krea import KreaImageGenProvider
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert "KREA_API_KEY" in result["error"]
assert result["error_type"] == "auth_required"
def test_empty_prompt(self):
from plugins.image_gen.krea import KreaImageGenProvider
result = KreaImageGenProvider().generate(prompt=" ")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_successful_generation(self):
"""Happy path: submit → one poll → completed → URL downloaded."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job("https://krea.cdn/result.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/krea_krea-2-medium_test.png"),
) as mock_save, \
patch("plugins.image_gen.krea.time.sleep"): # skip real waits
result = KreaImageGenProvider().generate(prompt="A cinematic lamp")
assert result["success"] is True
assert result["image"] == "/tmp/krea_krea-2-medium_test.png"
assert result["provider"] == "krea"
assert result["model"] == "krea-2-medium"
assert result["aspect_ratio"] == "landscape"
assert result["job_id"] == "00000000-0000-0000-0000-000000000abc"
assert result["resolution"] == "1K"
assert result["creativity"] == "medium"
# Submit hit the medium endpoint
post_url = mock_post.call_args[0][0]
assert post_url.endswith("/generate/image/krea/krea-2/medium")
# Poll hit /jobs/{job_id}
poll_url = mock_get.call_args[0][0]
assert "/jobs/00000000-0000-0000-0000-000000000abc" in poll_url
# URL was materialised once
mock_save.assert_called_once()
def test_large_model_routes_to_large_endpoint(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test")
post_url = mock_post.call_args[0][0]
assert post_url.endswith("/generate/image/krea/krea-2/large")
def test_aspect_ratio_mapping(self):
"""Hermes 'square' must map to Krea '1:1' in the wire payload."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test", aspect_ratio="square")
payload = mock_post.call_args.kwargs["json"]
assert payload["aspect_ratio"] == "1:1"
assert payload["resolution"] == "1K"
def test_auth_header(self):
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test")
headers = mock_post.call_args.kwargs["headers"]
assert headers["Authorization"] == "Bearer test-key-12345"
assert headers["Content-Type"] == "application/json"
def test_passthrough_seed_styles_moodboards(self):
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(
prompt="test",
seed=42,
styles=[{"id": "lora-1", "strength": 0.7}],
moodboards=[{"url": "https://x.com/mood.png"}, {"url": "https://x.com/mood2.png"}],
image_style_references=[{"url": f"https://x.com/{i}.png"} for i in range(15)],
creativity="high",
)
payload = mock_post.call_args.kwargs["json"]
assert payload["seed"] == 42
assert payload["styles"] == [{"id": "lora-1", "strength": 0.7}]
assert len(payload["moodboards"]) == 1 # capped at 1
assert len(payload["image_style_references"]) == 10 # capped at 10
assert payload["creativity"] == "high"
def test_unknown_kwargs_ignored(self):
"""Forward-compat: unknown kwargs must not break generate()."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(
prompt="test",
fictional_param="should be ignored",
num_images=4,
)
assert result["success"] is True
# ---------------------------------------------------------------------------
# Generate — error paths
# ---------------------------------------------------------------------------
class TestGenerateErrors:
def test_submit_http_error(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
resp = req_lib.Response()
resp.status_code = 401
resp._content = b'{"error": {"message": "Invalid API key"}}'
resp.headers["Content-Type"] = "application/json"
resp.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=resp)
)
with patch("plugins.image_gen.krea.requests.post", return_value=resp):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "401" in result["error"]
assert "Invalid API key" in result["error"]
def test_submit_timeout(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
with patch(
"plugins.image_gen.krea.requests.post", side_effect=req_lib.Timeout()
):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "timeout"
def test_submit_connection_error(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
with patch(
"plugins.image_gen.krea.requests.post",
side_effect=req_lib.ConnectionError("dns nope"),
):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "connection_error"
def test_submit_missing_job_id(self):
from plugins.image_gen.krea import KreaImageGenProvider
bad_submit = MagicMock()
bad_submit.status_code = 200
bad_submit.raise_for_status = MagicMock()
bad_submit.json.return_value = {"status": "queued"}
with patch("plugins.image_gen.krea.requests.post", return_value=bad_submit):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "invalid_response"
assert "job_id" in result["error"]
def test_job_failed(self):
from plugins.image_gen.krea import KreaImageGenProvider
failed = {
"job_id": "abc",
"status": "failed",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"error": "NSFW content"},
}
submit = _submit_response()
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(failed),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "NSFW" in result["error"]
def test_job_cancelled(self):
from plugins.image_gen.krea import KreaImageGenProvider
cancelled = {
"job_id": "abc",
"status": "cancelled",
"completed_at": "2026-05-27T00:01:00Z",
"result": {},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(cancelled),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "cancelled"
def test_completed_but_missing_urls(self):
from plugins.image_gen.krea import KreaImageGenProvider
completed_empty = {
"job_id": "abc",
"status": "completed",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"urls": []},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(completed_empty),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_url_download_failure_falls_back_to_bare_url(self):
"""Mirror of xAI behaviour — if local cache fails, return the URL."""
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
url = "https://krea.cdn/expired-soon.png"
submit = _submit_response()
poll = _poll_response(_completed_job(url))
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
side_effect=req_lib.HTTPError("404"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert result["image"] == url
def test_polling_picks_up_completed_at_with_unknown_status(self):
"""``completed_at`` set + unrecognised pending status → still terminal."""
from plugins.image_gen.krea import KreaImageGenProvider
# Use a status value that is NOT in our terminal set ("intermediate-complete")
# but with completed_at populated — Krea's spec says completed_at is the
# canonical terminal marker.
oddball = {
"job_id": "abc",
"status": "intermediate-complete",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"urls": ["https://krea.cdn/done.png"]},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(oddball),
), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
class TestPollRetryPolicy:
"""Polling fail-fast on permanent 4xx, retry on transient 5xx/429."""
def _http_error_response(self, status: int):
import requests as req_lib
resp = req_lib.Response()
resp.status_code = status
resp._content = b'{"error": "boom"}'
resp.headers["Content-Type"] = "application/json"
resp.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=resp)
)
return resp
def test_poll_fails_fast_on_401(self):
"""Auth failure mid-poll should not wait the 180s deadline."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(401)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "401" in result["error"]
# One call — no retry on permanent auth failure.
assert mock_get.call_count == 1
def test_poll_fails_fast_on_404(self):
"""Missing job (404) should surface immediately, not retry for 180s."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(404)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "404" in result["error"]
assert mock_get.call_count == 1
def test_poll_fails_fast_on_403(self):
"""Billing/permission failure (403) should not retry."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(403)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert mock_get.call_count == 1
def test_poll_retries_on_503_then_succeeds(self):
"""Transient 5xx should retry and eventually surface a completion."""
from plugins.image_gen.krea import KreaImageGenProvider
flaky = self._http_error_response(503)
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
side_effect=[flaky, flaky, good],
) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert mock_get.call_count == 3
def test_poll_retries_on_429(self):
"""Rate-limit (429) is in the retryable set."""
from plugins.image_gen.krea import KreaImageGenProvider
rate_limited = self._http_error_response(429)
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
side_effect=[rate_limited, good],
) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert mock_get.call_count == 2
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
class TestRegistration:
def test_register(self):
from plugins.image_gen.krea import KreaImageGenProvider, register
mock_ctx = MagicMock()
register(mock_ctx)
mock_ctx.register_image_gen_provider.assert_called_once()
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
assert isinstance(provider, KreaImageGenProvider)
assert provider.name == "krea"
@@ -0,0 +1,236 @@
"""Tests for the bundled ``openai-codex`` image_gen plugin.
Mirrors ``test_openai_provider.py`` but targets the standalone
Codex/ChatGPT-OAuth-backed provider that uses the Responses
``image_generation`` tool path instead of the ``images.generate`` REST
endpoint.
"""
from __future__ import annotations
import importlib
from pathlib import Path
import pytest
# The plugin directory uses a hyphen, which is not a valid Python identifier
# for the dotted-import form. Load it via importlib so tests don't need to
# touch sys.path or rename the directory.
codex_plugin = importlib.import_module("plugins.image_gen.openai-codex")
# 1×1 transparent PNG — valid bytes for save_b64_image()
_PNG_HEX = (
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
"ae426082"
)
def _b64_png() -> str:
import base64
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
@pytest.fixture(autouse=True)
def _tmp_hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
@pytest.fixture
def provider(monkeypatch):
# Codex plugin is API-key-independent; clear it to make the test honest.
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
return codex_plugin.OpenAICodexImageGenProvider()
# ── Metadata ────────────────────────────────────────────────────────────────
class TestMetadata:
def test_name(self, provider):
assert provider.name == "openai-codex"
def test_display_name(self, provider):
assert provider.display_name == "OpenAI (Codex auth)"
def test_default_model(self, provider):
assert provider.default_model() == "gpt-image-2-medium"
def test_list_models_three_tiers(self, provider):
ids = [m["id"] for m in provider.list_models()]
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
def test_setup_schema_has_no_required_env_vars(self, provider):
schema = provider.get_setup_schema()
assert schema["env_vars"] == []
assert schema["badge"] == "free"
# ── Availability ────────────────────────────────────────────────────────────
class TestAvailability:
def test_unavailable_without_codex_token(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
def test_available_with_codex_token(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is True
def test_openai_api_key_alone_is_not_enough(self, monkeypatch):
# Codex plugin is intentionally orthogonal to the API-key plugin —
# the API key alone must NOT make it appear available.
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
# ── Generate ────────────────────────────────────────────────────────────────
class TestGenerate:
def test_returns_auth_error_without_codex_token(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_returns_invalid_argument_for_empty_prompt(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
result = provider.generate(" ")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_generate_uses_codex_stream_path(self, provider, monkeypatch, tmp_path):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: _b64_png())
result = provider.generate("a cat", aspect_ratio="landscape")
assert result["success"] is True
assert result["model"] == "gpt-image-2-medium"
assert result["provider"] == "openai-codex"
assert result["quality"] == "medium"
saved = Path(result["image"])
assert saved.exists()
assert saved.parent == tmp_path / "cache" / "images"
# Filename prefix differs from the API-key plugin so cache audits can
# tell the two backends apart.
assert saved.name.startswith("openai_codex_")
def test_codex_stream_request_shape(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
captured = {}
def _collect(token, *, prompt, size, quality):
captured.update(codex_plugin._build_responses_payload(
prompt=prompt,
size=size,
quality=quality,
))
return _b64_png()
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _collect)
result = provider.generate("a cat", aspect_ratio="portrait")
assert result["success"] is True
assert captured["model"] == "gpt-5.5"
assert captured["store"] is False
assert captured["input"][0]["type"] == "message"
assert captured["input"][0]["role"] == "user"
assert captured["input"][0]["content"][0]["type"] == "input_text"
assert captured["tool_choice"]["type"] == "allowed_tools"
assert captured["tool_choice"]["mode"] == "required"
assert captured["tool_choice"]["tools"] == [{"type": "image_generation"}]
tool = captured["tools"][0]
assert tool["type"] == "image_generation"
assert tool["model"] == "gpt-image-2"
assert tool["quality"] == "medium"
assert tool["size"] == "1024x1536"
assert tool["output_format"] == "png"
assert tool["background"] == "opaque"
assert tool["partial_images"] == 1
def test_partial_image_event_used_when_done_missing(self):
"""If output_item.done is missing, partial_image_b64 is accepted."""
payload = {
"type": "response.image_generation_call.partial_image",
"partial_image_b64": _b64_png(),
}
assert codex_plugin._extract_image_b64(payload) == _b64_png()
def test_sse_parser_handles_event_and_data_lines(self):
class _Response:
def iter_lines(self):
return iter([
"event: response.output_item.done",
'data: {"item": {"type": "image_generation_call", "result": "abc"}}',
"",
])
events = list(codex_plugin._iter_sse_json(_Response()))
assert events == [{
"type": "response.output_item.done",
"item": {"type": "image_generation_call", "result": "abc"},
}]
def test_final_response_sweep_recovers_image(self):
"""Completed response output is found by recursive payload scanning."""
payload = {
"type": "response.completed",
"response": {
"output": [{
"type": "image_generation_call",
"status": "completed",
"id": "ig_final",
"result": _b64_png(),
}],
},
}
assert codex_plugin._extract_image_b64(payload) == _b64_png()
def test_empty_response_returns_error(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: None)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_stream_exception_returns_api_error(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
def _boom(*args, **kwargs):
raise RuntimeError("cloudflare 403")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _boom)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "cloudflare 403" in result["error"]
# ── Plugin entry point ──────────────────────────────────────────────────────
class TestRegistration:
def test_register_calls_register_image_gen_provider(self):
registered = []
class _Ctx:
def register_image_gen_provider(self, prov):
registered.append(prov)
codex_plugin.register(_Ctx())
assert len(registered) == 1
assert registered[0].name == "openai-codex"
@@ -0,0 +1,271 @@
"""Tests for the bundled OpenAI image_gen plugin (gpt-image-2, three tiers)."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import plugins.image_gen.openai as openai_plugin
# 1×1 transparent PNG — valid bytes for save_b64_image()
_PNG_HEX = (
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
"ae426082"
)
def _b64_png() -> str:
import base64
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
def _fake_response(*, b64=None, url=None, revised_prompt=None):
item = SimpleNamespace(b64_json=b64, url=url, revised_prompt=revised_prompt)
return SimpleNamespace(data=[item])
@pytest.fixture(autouse=True)
def _tmp_hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
@pytest.fixture
def provider(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
return openai_plugin.OpenAIImageGenProvider()
def _patched_openai(fake_client: MagicMock):
fake_openai = MagicMock()
fake_openai.OpenAI.return_value = fake_client
return patch.dict("sys.modules", {"openai": fake_openai})
# ── Metadata ────────────────────────────────────────────────────────────────
class TestMetadata:
def test_name(self, provider):
assert provider.name == "openai"
def test_default_model(self, provider):
assert provider.default_model() == "gpt-image-2-medium"
def test_list_models_three_tiers(self, provider):
ids = [m["id"] for m in provider.list_models()]
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
def test_catalog_entries_have_display_speed_strengths(self, provider):
for entry in provider.list_models():
assert entry["display"].startswith("GPT Image 2")
assert entry["speed"]
assert entry["strengths"]
# ── Availability ────────────────────────────────────────────────────────────
class TestAvailability:
def test_no_api_key_unavailable(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
assert openai_plugin.OpenAIImageGenProvider().is_available() is False
def test_api_key_set_available(self, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test")
assert openai_plugin.OpenAIImageGenProvider().is_available() is True
# ── Model resolution ────────────────────────────────────────────────────────
class TestModelResolution:
def test_default_is_medium(self):
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-medium"
assert meta["quality"] == "medium"
def test_env_var_override(self, monkeypatch):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "gpt-image-2-high")
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-high"
assert meta["quality"] == "high"
def test_env_var_unknown_falls_back(self, monkeypatch):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "bogus-tier")
model_id, _ = openai_plugin._resolve_model()
assert model_id == openai_plugin.DEFAULT_MODEL
def test_config_openai_model(self, tmp_path):
import yaml
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"openai": {"model": "gpt-image-2-low"}}})
)
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-low"
assert meta["quality"] == "low"
def test_config_top_level_model(self, tmp_path):
"""``image_gen.model: gpt-image-2-high`` also works (top-level)."""
import yaml
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"model": "gpt-image-2-high"}})
)
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-high"
assert meta["quality"] == "high"
# ── Generate ────────────────────────────────────────────────────────────────
class TestGenerate:
def test_empty_prompt_rejected(self, provider):
result = provider.generate("", aspect_ratio="square")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
result = openai_plugin.OpenAIImageGenProvider().generate("a cat")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_b64_saves_to_cache(self, provider, tmp_path):
png_bytes = bytes.fromhex(_PNG_HEX)
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
result = provider.generate("a cat", aspect_ratio="landscape")
assert result["success"] is True
assert result["model"] == "gpt-image-2-medium"
assert result["aspect_ratio"] == "landscape"
assert result["provider"] == "openai"
assert result["quality"] == "medium"
saved = Path(result["image"])
assert saved.exists()
assert saved.parent == tmp_path / "cache" / "images"
assert saved.read_bytes() == png_bytes
call_kwargs = fake_client.images.generate.call_args.kwargs
# All tiers hit the single underlying API model.
assert call_kwargs["model"] == "gpt-image-2"
assert call_kwargs["quality"] == "medium"
assert call_kwargs["size"] == "1536x1024"
# gpt-image-2 rejects response_format — we must NOT send it.
assert "response_format" not in call_kwargs
@pytest.mark.parametrize("tier,expected_quality", [
("gpt-image-2-low", "low"),
("gpt-image-2-medium", "medium"),
("gpt-image-2-high", "high"),
])
def test_tier_maps_to_quality(self, provider, monkeypatch, tier, expected_quality):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", tier)
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["model"] == tier
assert result["quality"] == expected_quality
assert fake_client.images.generate.call_args.kwargs["quality"] == expected_quality
# Always the same underlying API model regardless of tier.
assert fake_client.images.generate.call_args.kwargs["model"] == "gpt-image-2"
@pytest.mark.parametrize("aspect,expected_size", [
("landscape", "1536x1024"),
("square", "1024x1024"),
("portrait", "1024x1536"),
])
def test_aspect_ratio_mapping(self, provider, aspect, expected_size):
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
provider.generate("a cat", aspect_ratio=aspect)
assert fake_client.images.generate.call_args.kwargs["size"] == expected_size
def test_revised_prompt_passed_through(self, provider):
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=_b64_png(), revised_prompt="A photo of a cat",
)
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["revised_prompt"] == "A photo of a cat"
def test_api_error_returns_error_response(self, provider):
fake_client = MagicMock()
fake_client.images.generate.side_effect = RuntimeError("boom")
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "boom" in result["error"]
def test_empty_response_data(self, provider):
fake_client = MagicMock()
fake_client.images.generate.return_value = SimpleNamespace(data=[])
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_url_response_is_cached_locally(self, provider):
"""OpenAI URL response (if API ever returns one) is cached locally.
Pre-fix this asserted the bare URL passed through; symmetric to the
xAI #26942 fix. Even though gpt-image-2 returns b64 today, every
``image_gen`` provider must guarantee the gateway gets a stable
file path so ephemeral signed URLs can't expire mid-flight.
"""
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=None, url="https://example.com/img.png",
)
with _patched_openai(fake_client), patch(
"plugins.image_gen.openai.save_url_image",
return_value=Path("/tmp/openai_gpt-image-2_20260524_000000_deadbeef.png"),
) as mock_save_url:
result = provider.generate("a cat")
assert result["success"] is True
assert result["image"].startswith("/")
assert "example.com" not in result["image"]
mock_save_url.assert_called_once()
def test_url_response_falls_back_to_bare_url_when_download_fails(self, provider):
"""Cache failure must not turn into a tool error — symmetric with xAI."""
import requests as req_lib
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=None, url="https://example.com/img.png",
)
with _patched_openai(fake_client), patch(
"plugins.image_gen.openai.save_url_image",
side_effect=req_lib.HTTPError("404 from CDN"),
):
result = provider.generate("a cat")
assert result["success"] is True
assert result["image"] == "https://example.com/img.png"
@@ -0,0 +1,336 @@
#!/usr/bin/env python3
"""Tests for xAI image generation provider."""
from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _fake_api_key(monkeypatch):
"""Ensure XAI_API_KEY is set for all tests."""
monkeypatch.setenv("XAI_API_KEY", "test-key-12345")
# ---------------------------------------------------------------------------
# Provider class tests
# ---------------------------------------------------------------------------
class TestXAIImageGenProvider:
def test_name(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.name == "xai"
def test_display_name(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.display_name == "xAI (Grok)"
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("XAI_API_KEY", "sk-xxx")
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.is_available() is True
def test_is_available_without_key(self, monkeypatch):
monkeypatch.delenv("XAI_API_KEY", raising=False)
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.is_available() is False
def test_list_models(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
models = provider.list_models()
assert len(models) >= 1
assert models[0]["id"] == "grok-imagine-image"
def test_default_model(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.default_model() == "grok-imagine-image"
def test_get_setup_schema(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
schema = provider.get_setup_schema()
assert schema["name"] == "xAI Grok Imagine (image)"
assert schema["badge"] == "paid"
# Auth resolution is delegated to the shared "xai_grok" post_setup
# hook so the picker doesn't blindly prompt for XAI_API_KEY when the
# user is already signed in via xAI Grok OAuth.
assert schema["env_vars"] == []
assert schema["post_setup"] == "xai_grok"
# ---------------------------------------------------------------------------
# Config tests
# ---------------------------------------------------------------------------
class TestConfig:
def test_default_model(self):
from plugins.image_gen.xai import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "grok-imagine-image"
def test_default_resolution(self):
from plugins.image_gen.xai import _resolve_resolution
assert _resolve_resolution() == "1k"
def test_custom_model(self, monkeypatch):
monkeypatch.setenv("XAI_IMAGE_MODEL", "grok-imagine-image")
from plugins.image_gen.xai import _resolve_model
model_id, _ = _resolve_model()
assert model_id == "grok-imagine-image"
# ---------------------------------------------------------------------------
# Generate tests
# ---------------------------------------------------------------------------
class TestGenerate:
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("XAI_API_KEY", raising=False)
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert "XAI_API_KEY" in result["error"]
def test_successful_generation(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"b64_json": "dGVzdC1pbWFnZS1kYXRh"}], # base64 "test-image-data"
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
with patch("plugins.image_gen.xai.save_b64_image", return_value="/tmp/test.png"):
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True
assert result["image"] == "/tmp/test.png"
assert result["provider"] == "xai"
assert result["model"] == "grok-imagine-image"
def test_successful_url_response(self):
"""xAI URL response is cached locally — #26942 contract.
Pre-fix this asserted ``result["image"] == "<the bare URL>"``, which
was exactly the bug: xAI's ``imgen.x.ai/xai-tmp-*`` URLs expire fast
and the gateway 404'd by ``send_photo`` time. Post-fix the URL
bytes are downloaded at tool-completion and the result carries an
absolute filesystem path the gateway can upload from.
"""
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
patch(
"plugins.image_gen.xai.save_url_image",
return_value=Path("/tmp/xai_grok-imagine-image_20260524_000000_deadbeef.jpg"),
) as mock_save_url:
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True
assert result["image"].startswith("/"), (
f"URL response must be cached to an absolute path, got {result['image']!r}"
)
assert "imgen.x.ai" not in result["image"], (
"ephemeral xAI URL must not leak into result.image — caller will 404"
)
# The downloader should have been called exactly once with the URL
# and an xai-prefixed cache filename.
mock_save_url.assert_called_once()
call_args, call_kwargs = mock_save_url.call_args
assert call_args[0] == "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"
assert call_kwargs.get("prefix", "").startswith("xai_")
def test_url_response_falls_back_to_bare_url_when_download_fails(self):
"""If caching the URL fails (network blip, 404 in-flight), the
provider must NOT hard-error fall through to returning the bare
URL so the agent surface at least sees *something*. The gateway's
existing URL-send fallback then has a chance to succeed; if it
too 404s, the user gets the original (now legible) error rather
than an opaque "image generation failed" tool result.
"""
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
patch(
"plugins.image_gen.xai.save_url_image",
side_effect=req_lib.HTTPError("404 from CDN"),
):
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True, (
"Cache failure must not turn into a tool error — gateway gets a chance to retry"
)
assert result["image"] == "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"
def test_api_error(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 401
mock_resp.text = "Unauthorized"
mock_resp.json.return_value = {"error": {"message": "Invalid API key"}}
mock_resp.raise_for_status.side_effect = req_lib.HTTPError(response=mock_resp)
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
def test_api_error_preserves_real_response_status(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
response = req_lib.Response()
response.status_code = 401
response._content = json.dumps({"error": {"message": "Invalid API key"}}).encode()
response.headers["Content-Type"] = "application/json"
response.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=response)
)
with patch("plugins.image_gen.xai.requests.post", return_value=response):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "xAI image generation failed (401): Invalid API key" in result["error"]
def test_timeout(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
with patch("plugins.image_gen.xai.requests.post", side_effect=req_lib.Timeout()):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "timeout"
def test_empty_response(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {"data": []}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_auth_header(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://xai.image/test.png"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
provider = XAIImageGenProvider()
provider.generate(prompt="test")
call_args = mock_post.call_args
headers = call_args.kwargs.get("headers") or call_args[1].get("headers")
assert "Bearer test-key-12345" in headers["Authorization"]
assert "Hermes-Agent" in headers["User-Agent"]
def test_payload_resolution_is_literal_1k_or_2k(self):
"""Regression: xAI API rejects numeric resolutions ("1024"/"2048") with 422.
The endpoint expects the literal strings "1k" or "2k". Ensure the wire
payload carries that literal not a numeric mapping. See PR #18678.
"""
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {"data": [{"url": "https://xai.image/test.png"}]}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
provider = XAIImageGenProvider()
provider.generate(prompt="test")
payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json")
assert payload["resolution"] in {"1k", "2k"}, (
f"resolution must be the literal '1k' or '2k', got {payload['resolution']!r}"
)
# ---------------------------------------------------------------------------
# Registration test
# ---------------------------------------------------------------------------
class TestRegistration:
def test_register(self):
from plugins.image_gen.xai import XAIImageGenProvider, register
mock_ctx = MagicMock()
register(mock_ctx)
mock_ctx.register_image_gen_provider.assert_called_once()
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
assert isinstance(provider, XAIImageGenProvider)
assert provider.name == "xai"