Hermes-agent

plugins/web/__init__.py

@@ -0,0 +1,7 @@
+# Bundled web search providers — plugins/web/.
+#
+# Each subdirectory follows the image_gen plugin layout:
+#   plugins/web/<name>/{plugin.yaml, __init__.py, provider.py}
+#
+# They auto-load via kind: backend and register via
+# ctx.register_web_search_provider() into agent.web_search_registry.

plugins/web/brave_free/__init__.py

@@ -0,0 +1,14 @@
+"""Brave Search (free tier) plugin — bundled, auto-loaded.
+
+Mirrors the ``plugins/image_gen/openai/`` layout: ``provider.py`` holds the
+provider class, ``__init__.py::register(ctx)`` registers an instance.
+"""
+
+from __future__ import annotations
+
+from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Brave-free provider with the plugin context."""
+    ctx.register_web_search_provider(BraveFreeWebSearchProvider())

plugins/web/brave_free/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-brave-free
+version: 1.0.0
+description: "Brave Search (free tier) — web search via Brave's Data-for-Search API. Requires BRAVE_SEARCH_API_KEY (free signup at https://brave.com/search/api/, 2k queries/month)."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - brave-free

plugins/web/brave_free/provider.py

@@ -0,0 +1,137 @@
+"""Brave Search (free tier) — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider` (the
+plugin-facing ABC). The legacy in-tree module
+``tools.web_providers.brave_free`` was removed in the same commit that
+moved this code under ``plugins/``; this file is now the canonical
+implementation.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "brave-free"     # explicit per-capability
+      backend: "brave-free"            # shared fallback
+
+Auth env var::
+
+    BRAVE_SEARCH_API_KEY=...    # https://brave.com/search/api/ (free tier)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+_BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
+
+
+class BraveFreeWebSearchProvider(WebSearchProvider):
+    """Search-only Brave provider using the free-tier Data-for-Search API.
+
+    Free tier is 2,000 queries/month (1 qps). No content-extraction capability —
+    users pair this with Firecrawl/Tavily/Exa for ``web_extract``.
+    """
+
+    @property
+    def name(self) -> str:
+        # Hyphen form preserved for backward compat with the existing
+        # ``web.search_backend: "brave-free"`` config keys users have set.
+        return "brave-free"
+
+    @property
+    def display_name(self) -> str:
+        return "Brave Search (Free)"
+
+    def is_available(self) -> bool:
+        """Return True when ``BRAVE_SEARCH_API_KEY`` is set to a non-empty value."""
+        return bool(os.getenv("BRAVE_SEARCH_API_KEY", "").strip())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a search against the Brave Search API.
+
+        Returns ``{"success": True, "data": {"web": [{"title", "url", "description", "position"}]}}``
+        on success, or ``{"success": False, "error": str}`` on failure.
+        """
+        import httpx
+
+        api_key = os.getenv("BRAVE_SEARCH_API_KEY", "").strip()
+        if not api_key:
+            return {"success": False, "error": "BRAVE_SEARCH_API_KEY is not set"}
+
+        # Brave's `count` is capped at 20.
+        count = max(1, min(int(limit), 20))
+
+        try:
+            resp = httpx.get(
+                _BRAVE_ENDPOINT,
+                params={"q": query, "count": count},
+                headers={
+                    "X-Subscription-Token": api_key,
+                    "Accept": "application/json",
+                },
+                timeout=15,
+            )
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            logger.warning("Brave Search HTTP error: %s", exc)
+            return {
+                "success": False,
+                "error": f"Brave Search returned HTTP {exc.response.status_code}",
+            }
+        except httpx.RequestError as exc:
+            logger.warning("Brave Search request error: %s", exc)
+            return {"success": False, "error": f"Could not reach Brave Search: {exc}"}
+
+        try:
+            data = resp.json()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Brave Search response parse error: %s", exc)
+            return {"success": False, "error": "Could not parse Brave Search response as JSON"}
+
+        raw_results = (data.get("web") or {}).get("results", []) or []
+        truncated = raw_results[:limit]
+
+        web_results = [
+            {
+                "title": str(r.get("title", "")),
+                "url": str(r.get("url", "")),
+                "description": str(r.get("description", "")),
+                "position": i + 1,
+            }
+            for i, r in enumerate(truncated)
+        ]
+
+        logger.info(
+            "Brave Search '%s': %d results (from %d raw, limit %d)",
+            query,
+            len(web_results),
+            len(raw_results),
+            limit,
+        )
+
+        return {"success": True, "data": {"web": web_results}}
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Brave Search (Free)",
+            "badge": "free",
+            "tag": "Free-tier API key — 2k queries/mo, search only.",
+            "env_vars": [
+                {
+                    "key": "BRAVE_SEARCH_API_KEY",
+                    "prompt": "Brave Search API key (free tier)",
+                    "url": "https://brave.com/search/api/",
+                },
+            ],
+        }

plugins/web/ddgs/__init__.py

@@ -0,0 +1,15 @@
+"""DuckDuckGo search plugin — bundled, auto-loaded.
+
+Backed by the community ``ddgs`` Python package which scrapes DDG's HTML
+results page. No API key required, but the package itself must be installed
+(it's an optional dep — gated via :meth:`is_available`).
+"""
+
+from __future__ import annotations
+
+from plugins.web.ddgs.provider import DDGSWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the DDGS provider with the plugin context."""
+    ctx.register_web_search_provider(DDGSWebSearchProvider())

plugins/web/ddgs/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-ddgs
+version: 1.0.0
+description: "DuckDuckGo web search via the ddgs Python package — no API key required. Install with `pip install ddgs`."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - ddgs

plugins/web/ddgs/provider.py

@@ -0,0 +1,104 @@
+"""DuckDuckGo search — plugin form (via the ``ddgs`` package).
+
+Subclasses the plugin-facing :class:`agent.web_search_provider.WebSearchProvider`.
+The legacy in-tree module ``tools.web_providers.ddgs`` was removed in the
+same commit that moved this code under ``plugins/``; this file is now the
+canonical implementation.
+
+The ``ddgs`` package is an optional dependency. ``is_available()`` reflects
+whether the package is importable; the plugin still registers either way so
+``hermes tools`` can prompt the user to install it.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+
+class DDGSWebSearchProvider(WebSearchProvider):
+    """DuckDuckGo HTML-scrape search provider.
+
+    No API key needed. Rate limits are enforced server-side by DuckDuckGo;
+    the provider surfaces ``DuckDuckGoSearchException`` and other ddgs errors
+    as ``{"success": False, "error": ...}`` rather than raising.
+    """
+
+    @property
+    def name(self) -> str:
+        return "ddgs"
+
+    @property
+    def display_name(self) -> str:
+        return "DuckDuckGo (ddgs)"
+
+    def is_available(self) -> bool:
+        """Return True when the ``ddgs`` package is importable.
+
+        Probes the import once; cheap because Python caches the import. Must
+        NOT perform network I/O — runs at tool-registration time and on every
+        ``hermes tools`` paint.
+        """
+        try:
+            import ddgs  # noqa: F401
+
+            return True
+        except ImportError:
+            return False
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a DuckDuckGo search and return normalized results."""
+        try:
+            from ddgs import DDGS  # type: ignore
+        except ImportError:
+            return {
+                "success": False,
+                "error": "ddgs package is not installed — run `pip install ddgs`",
+            }
+
+        # DDGS().text yields at most `max_results` items; we cap defensively
+        # in case the package ignores the hint.
+        safe_limit = max(1, int(limit))
+
+        try:
+            web_results = []
+            with DDGS() as client:
+                for i, hit in enumerate(client.text(query, max_results=safe_limit)):
+                    if i >= safe_limit:
+                        break
+                    url = str(hit.get("href") or hit.get("url") or "")
+                    web_results.append(
+                        {
+                            "title": str(hit.get("title", "")),
+                            "url": url,
+                            "description": str(hit.get("body", "")),
+                            "position": i + 1,
+                        }
+                    )
+        except Exception as exc:  # noqa: BLE001 — ddgs raises its own exceptions
+            logger.warning("DDGS search error: %s", exc)
+            return {"success": False, "error": f"DuckDuckGo search failed: {exc}"}
+
+        logger.info("DDGS search '%s': %d results (limit %d)", query, len(web_results), limit)
+        return {"success": True, "data": {"web": web_results}}
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "DuckDuckGo (ddgs)",
+            "badge": "free · no key · search only",
+            "tag": "Search via the ddgs Python package — no API key (pair with any extract provider)",
+            "env_vars": [],
+            # Trigger `_run_post_setup("ddgs")` after the user picks this row
+            # so the ddgs Python package gets pip-installed on first selection.
+            "post_setup": "ddgs",
+        }

plugins/web/exa/__init__.py

@@ -0,0 +1,15 @@
+"""Exa web search + extract plugin — bundled, auto-loaded.
+
+Backed by the official Exa SDK (``exa-py``). Both search and extract are
+sync; the dispatcher in :mod:`tools.web_tools` handles the wrap when the
+caller is async.
+"""
+
+from __future__ import annotations
+
+from plugins.web.exa.provider import ExaWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Exa provider with the plugin context."""
+    ctx.register_web_search_provider(ExaWebSearchProvider())

plugins/web/exa/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-exa
+version: 1.0.0
+description: "Exa web search and content extraction. Requires EXA_API_KEY — sign up at https://exa.ai."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - exa

plugins/web/exa/provider.py

@@ -0,0 +1,212 @@
+"""Exa web search + content extraction — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses the
+official Exa SDK (``exa-py``) which is lazy-loaded via
+:func:`tools.lazy_deps.ensure` so that cold-start CLI users don't pay the
+SDK import cost when Exa isn't configured.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "exa"      # explicit per-capability
+      extract_backend: "exa"     # explicit per-capability
+      backend: "exa"             # shared fallback for both
+
+Env var::
+
+    EXA_API_KEY=...    # https://exa.ai (paid tier; free trial available)
+
+The previous in-tree implementation lived at
+``tools.web_tools._exa_search`` / ``_exa_extract``; this file is the
+canonical replacement. Behavior is bit-for-bit identical aside from the
+ABC method-name change.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, List
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+# Module-level note: the canonical ``_exa_client`` cache slot lives on
+# :mod:`tools.web_tools` so tests that do ``tools.web_tools._exa_client =
+# None`` between cases see fresh state. The plugin reads/writes through
+# that public module (see :func:`_get_exa_client`).
+
+
+def _get_exa_client() -> Any:
+    """Lazy-import and cache an Exa SDK client.
+
+    Cache lives on :mod:`tools.web_tools` (as ``_exa_client``) so unit
+    tests that reset that name between cases keep working. Raises
+    ``ValueError`` when ``EXA_API_KEY`` is unset.
+    """
+    import tools.web_tools as _wt
+
+    cached = getattr(_wt, "_exa_client", None)
+    if cached is not None:
+        return cached
+
+    api_key = os.getenv("EXA_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "EXA_API_KEY environment variable not set. "
+            "Get your API key at https://exa.ai"
+        )
+
+    try:
+        from tools.lazy_deps import ensure as _lazy_ensure
+
+        _lazy_ensure("search.exa", prompt=False)
+    except ImportError:
+        pass
+    except Exception as exc:  # noqa: BLE001 — lazy_deps surfaces install hints
+        raise ImportError(str(exc))
+
+    from exa_py import Exa  # noqa: WPS433 — deliberately lazy
+
+    client = Exa(api_key=api_key)
+    client.headers["x-exa-integration"] = "hermes-agent"
+    _wt._exa_client = client
+    return client
+
+
+def _reset_client_for_tests() -> None:
+    """Drop the cached Exa client so tests can re-instantiate cleanly."""
+    import tools.web_tools as _wt
+
+    _wt._exa_client = None
+
+
+class ExaWebSearchProvider(WebSearchProvider):
+    """Exa search + extract provider.
+
+    Both methods are sync — Exa's SDK is sync-only. The web_extract_tool
+    dispatcher wraps sync extracts via ``asyncio.to_thread`` when it
+    needs to keep the event loop responsive.
+    """
+
+    @property
+    def name(self) -> str:
+        return "exa"
+
+    @property
+    def display_name(self) -> str:
+        return "Exa"
+
+    def is_available(self) -> bool:
+        """Return True when ``EXA_API_KEY`` is set to a non-empty value."""
+        return bool(os.getenv("EXA_API_KEY", "").strip())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute an Exa search.
+
+        Returns ``{"success": True, "data": {"web": [{...}, ...]}}`` on
+        success, ``{"success": False, "error": str}`` on failure (incl.
+        missing API key and SDK install errors).
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            logger.info("Exa search: '%s' (limit=%d)", query, limit)
+            response = _get_exa_client().search(
+                query,
+                num_results=limit,
+                contents={"highlights": True},
+            )
+
+            web_results = []
+            for i, result in enumerate(response.results or []):
+                highlights = result.highlights or []
+                web_results.append(
+                    {
+                        "url": result.url or "",
+                        "title": result.title or "",
+                        "description": " ".join(highlights) if highlights else "",
+                        "position": i + 1,
+                    }
+                )
+
+            return {"success": True, "data": {"web": web_results}}
+        except ValueError as exc:
+            # Raised by _get_exa_client when EXA_API_KEY missing
+            return {"success": False, "error": str(exc)}
+        except ImportError as exc:
+            return {"success": False, "error": f"Exa SDK not installed: {exc}"}
+        except Exception as exc:  # noqa: BLE001 — surface as failure
+            logger.warning("Exa search error: %s", exc)
+            return {"success": False, "error": f"Exa search failed: {exc}"}
+
+    def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via Exa.
+
+        Returns a list of result dicts shaped for the legacy LLM
+        post-processing pipeline. On per-URL or whole-batch failure,
+        results carry an ``error`` field rather than raising.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            logger.info("Exa extract: %d URL(s)", len(urls))
+            response = _get_exa_client().get_contents(urls, text=True)
+
+            results: List[Dict[str, Any]] = []
+            for result in response.results or []:
+                content = result.text or ""
+                url = result.url or ""
+                title = result.title or ""
+                results.append(
+                    {
+                        "url": url,
+                        "title": title,
+                        "content": content,
+                        "raw_content": content,
+                        "metadata": {"sourceURL": url, "title": title},
+                    }
+                )
+            return results
+        except ValueError as exc:
+            return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
+        except ImportError as exc:
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Exa SDK not installed: {exc}"}
+                for u in urls
+            ]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Exa extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Exa extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Exa",
+            "badge": "paid",
+            "tag": "Semantic + neural web search with content extraction.",
+            "env_vars": [
+                {
+                    "key": "EXA_API_KEY",
+                    "prompt": "Exa API key",
+                    "url": "https://exa.ai",
+                },
+            ],
+        }

plugins/web/firecrawl/__init__.py

@@ -0,0 +1,28 @@
+"""Firecrawl web search + extract plugin — bundled, auto-loaded.
+
+Largest single plugin in this PR. Captures everything the previous
+inline implementation in tools/web_tools.py did:
+
+  - Lazy import of the firecrawl SDK (~200ms cold-start cost) via a
+    callable proxy that defers the actual import to first use.
+  - Dual client paths: direct (FIRECRAWL_API_KEY / FIRECRAWL_API_URL)
+    OR Nous-hosted tool-gateway routing for subscribers, with
+    web.use_gateway as the tie-breaker.
+  - Per-URL scrape loop with 60s timeout, SSRF re-check after redirect,
+    website-policy gating, and format-aware content selection.
+  - Robust response shape normalization across SDK / direct API /
+    gateway variants (search returns differ by transport).
+
+The plugin re-exports ``Firecrawl`` (the lazy proxy) and
+``check_firecrawl_api_key`` for backward-compatibility with tests and
+external code that imports those names from ``tools.web_tools``.
+"""
+
+from __future__ import annotations
+
+from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Firecrawl provider with the plugin context."""
+    ctx.register_web_search_provider(FirecrawlWebSearchProvider())

plugins/web/firecrawl/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-firecrawl
+version: 1.0.0
+description: "Firecrawl web search + content extraction. Supports direct API and Nous-hosted tool-gateway routing for subscribers. Requires FIRECRAWL_API_KEY (or FIRECRAWL_API_URL for self-hosted), or an active Nous subscription with FIRECRAWL_GATEWAY_URL."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - firecrawl

plugins/web/firecrawl/provider.py

@@ -0,0 +1,594 @@
+"""Firecrawl web search + extract — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. This is
+the largest provider migrated in this PR; it captures the full inline
+firecrawl implementation that previously lived in tools/web_tools.py:
+
+  - :data:`Firecrawl` lazy proxy that defers the ~200ms SDK import to
+    first use (re-exported by tools.web_tools for backward compat with
+    existing tests that mock that name).
+  - :func:`_get_firecrawl_client` with direct + managed-gateway dual
+    mode, controlled by ``web.use_gateway`` config when both are
+    configured.
+  - :func:`check_firecrawl_api_key` re-exported (tests + tools_config
+    setup hint depend on this name living in tools.web_tools).
+  - :func:`_extract_web_search_results` / :func:`_extract_scrape_payload`
+    response-shape normalizers that handle SDK / direct API / gateway
+    response variants.
+  - Per-URL extract loop with 60s timeout, redirect-aware SSRF re-check,
+    website-policy gating, and format-aware content selection.
+
+Async note: the underlying SDK is sync. ``extract()`` is declared
+``async def`` because it performs per-URL I/O that benefits from
+running in an executor; the implementation wraps each scrape in
+:func:`asyncio.to_thread` with :func:`asyncio.wait_for(timeout=60)` to
+guard against hung fetches.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "firecrawl"     # explicit per-capability
+      extract_backend: "firecrawl"    # explicit per-capability
+      backend: "firecrawl"            # shared fallback (default)
+      use_gateway: false              # prefer managed gateway when both
+                                      # direct + gateway credentials exist
+
+Env vars::
+
+    FIRECRAWL_API_KEY=...            # direct cloud auth
+    FIRECRAWL_API_URL=...            # self-hosted Firecrawl
+    FIRECRAWL_GATEWAY_URL=...        # Nous tool-gateway (subscribers)
+    TOOL_GATEWAY_DOMAIN=...          # alternate gateway env
+    TOOL_GATEWAY_SCHEME=...
+    TOOL_GATEWAY_USER_TOKEN=...
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from agent.web_search_provider import WebSearchProvider
+from tools.website_policy import check_website_access
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Lazy Firecrawl SDK proxy
+# ---------------------------------------------------------------------------
+# The firecrawl SDK pulls ~200ms of imports (httpcore, firecrawl.v1/v2 type
+# trees) on a cold CLI. We only need it when the backend is actually
+# "firecrawl", so defer the import to first use via a callable proxy.
+#
+# Tests that do ``patch("tools.web_tools.Firecrawl", ...)`` continue to
+# work because tools/web_tools.py re-exports ``Firecrawl`` from this
+# module — so the patched name still references the same proxy instance.
+
+if TYPE_CHECKING:
+    from firecrawl import Firecrawl as FirecrawlSDK  # noqa: F401 — type hints only
+
+_FIRECRAWL_CLS_CACHE: Optional[type] = None
+
+
+def _load_firecrawl_cls() -> type:
+    """Import and cache ``firecrawl.Firecrawl``."""
+    global _FIRECRAWL_CLS_CACHE
+    if _FIRECRAWL_CLS_CACHE is None:
+        try:
+            from tools.lazy_deps import ensure as _lazy_ensure
+
+            _lazy_ensure("search.firecrawl", prompt=False)
+        except ImportError:
+            pass
+        except Exception as exc:  # noqa: BLE001 — surface install hint
+            raise ImportError(str(exc))
+        from firecrawl import Firecrawl as _cls  # noqa: WPS433 — deliberately lazy
+
+        _FIRECRAWL_CLS_CACHE = _cls
+    return _FIRECRAWL_CLS_CACHE
+
+
+class _FirecrawlProxy:
+    """Callable proxy that looks like ``firecrawl.Firecrawl`` but imports lazily."""
+
+    __slots__ = ()
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        return _load_firecrawl_cls()(*args, **kwargs)
+
+    def __instancecheck__(self, obj: Any) -> bool:
+        return isinstance(obj, _load_firecrawl_cls())
+
+    def __repr__(self) -> str:
+        return "<lazy firecrawl.Firecrawl proxy>"
+
+
+Firecrawl = _FirecrawlProxy()
+
+
+# ---------------------------------------------------------------------------
+# Client construction (direct vs managed-gateway)
+# ---------------------------------------------------------------------------
+#
+# The canonical cache slots live on :mod:`tools.web_tools` so tests that do
+# ``tools.web_tools._firecrawl_client = None`` between cases see fresh
+# state. The plugin reads/writes through that public module — see
+# :func:`_get_firecrawl_client` below.
+
+
+def _get_direct_firecrawl_config() -> Optional[tuple]:
+    """Return explicit direct Firecrawl kwargs + cache key, or None when unset."""
+    api_key = os.getenv("FIRECRAWL_API_KEY", "").strip()
+    api_url = os.getenv("FIRECRAWL_API_URL", "").strip().rstrip("/")
+
+    if not api_key and not api_url:
+        return None
+
+    kwargs: Dict[str, str] = {}
+    if api_key:
+        kwargs["api_key"] = api_key
+    if api_url:
+        kwargs["api_url"] = api_url
+
+    return kwargs, ("direct", api_url or None, api_key or None)
+
+
+def _get_firecrawl_gateway_url() -> str:
+    """Return the configured Firecrawl gateway URL."""
+    import tools.web_tools as _wt
+
+    return _wt.build_vendor_gateway_url("firecrawl")
+
+
+def _is_tool_gateway_ready() -> bool:
+    """Return True when gateway URL + Nous Subscriber token are available.
+
+    Reads ``peek_nous_access_token`` and ``resolve_managed_tool_gateway``
+    via :mod:`tools.web_tools` rather than direct imports, so unit tests
+    that ``patch("tools.web_tools._peek_nous_access_token", ...)`` see
+    their patches honored. The names are re-exported on
+    :mod:`tools.web_tools` for exactly this reason.
+    """
+    import tools.web_tools as _wt
+
+    return _wt.resolve_managed_tool_gateway(
+        "firecrawl", token_reader=_wt._peek_nous_access_token
+    ) is not None
+
+
+def _has_direct_firecrawl_config() -> bool:
+    """Return True when direct Firecrawl config is explicitly configured."""
+    return _get_direct_firecrawl_config() is not None
+
+
+def check_firecrawl_api_key() -> bool:
+    """Return True when Firecrawl backend (direct or gateway) is usable.
+
+    Re-exported by :mod:`tools.web_tools` for backward compatibility with
+    existing tests and the ``hermes tools`` setup flow.
+    """
+    return _has_direct_firecrawl_config() or _is_tool_gateway_ready()
+
+
+def _firecrawl_backend_help_suffix() -> str:
+    """Return optional managed-gateway guidance for Firecrawl help text."""
+    import tools.web_tools as _wt
+
+    if not _wt.managed_nous_tools_enabled():
+        return ""
+    return (
+        ", or use the Nous Tool Gateway via your subscription "
+        "(FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)"
+    )
+
+
+def _raise_web_backend_configuration_error() -> None:
+    """Raise a clear error for unsupported web backend configuration."""
+    import tools.web_tools as _wt
+
+    message = (
+        "Web tools are not configured. "
+        "Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL "
+        "for a self-hosted Firecrawl instance."
+    )
+    if _wt.managed_nous_tools_enabled():
+        message += (
+            " With your Nous subscription you can also use the Tool Gateway. "
+            "run `hermes tools` and select Nous Subscription as the web provider."
+        )
+    else:
+        message += " " + _wt.nous_tool_gateway_unavailable_message(
+            "managed Firecrawl web tools",
+        )
+    raise ValueError(message)
+
+
+def _get_firecrawl_client() -> Any:
+    """Get or create the cached Firecrawl client.
+
+    When ``web.use_gateway`` is set in config, the managed Tool Gateway is
+    preferred even if direct Firecrawl credentials are present. Otherwise
+    direct Firecrawl takes precedence when explicitly configured.
+
+    Raises ValueError when neither path is usable.
+
+    The cached client is stored on :mod:`tools.web_tools` (as
+    ``_firecrawl_client`` and ``_firecrawl_client_config``) rather than on
+    this plugin module so that unit tests that reset the cache via
+    ``tools.web_tools._firecrawl_client = None`` keep working. Helper
+    functions (``prefers_gateway``, ``resolve_managed_tool_gateway``,
+    ``_read_nous_access_token``, ``Firecrawl``) are also looked up via
+    :mod:`tools.web_tools` for the same reason — see
+    :func:`_is_tool_gateway_ready`.
+    """
+    import tools.web_tools as _wt
+
+    direct_config = _get_direct_firecrawl_config()
+    if direct_config is not None and not _wt.prefers_gateway("web"):
+        kwargs, client_config = direct_config
+    else:
+        managed_gateway = _wt.resolve_managed_tool_gateway(
+            "firecrawl", token_reader=_wt._read_nous_access_token
+        )
+        if managed_gateway is None:
+            logger.error(
+                "Firecrawl client initialization failed: "
+                "missing direct config and tool-gateway auth."
+            )
+            _raise_web_backend_configuration_error()
+
+        kwargs = {
+            "api_key": managed_gateway.nous_user_token,
+            "api_url": managed_gateway.gateway_origin,
+        }
+        client_config = (
+            "tool-gateway",
+            kwargs["api_url"],
+            managed_gateway.nous_user_token,
+        )
+
+    cached = getattr(_wt, "_firecrawl_client", None)
+    cached_config = getattr(_wt, "_firecrawl_client_config", None)
+    if cached is not None and cached_config == client_config:
+        return cached
+
+    # Construct via the re-exported Firecrawl proxy on tools.web_tools so
+    # unit tests patching ``tools.web_tools.Firecrawl`` see their mock.
+    _wt._firecrawl_client = _wt.Firecrawl(**kwargs)
+    _wt._firecrawl_client_config = client_config
+    return _wt._firecrawl_client
+
+
+def _reset_client_for_tests() -> None:
+    """Drop the cached Firecrawl client so tests can re-instantiate cleanly.
+
+    Clears the canonical slots on :mod:`tools.web_tools` (where
+    :func:`_get_firecrawl_client` reads/writes them).
+    """
+    import tools.web_tools as _wt
+
+    _wt._firecrawl_client = None
+    _wt._firecrawl_client_config = None
+
+
+# ---------------------------------------------------------------------------
+# Response shape normalization (SDK / direct / gateway differ)
+# ---------------------------------------------------------------------------
+
+
+def _to_plain_object(value: Any) -> Any:
+    """Convert SDK objects to plain python data structures when possible."""
+    if value is None:
+        return None
+
+    if isinstance(value, (dict, list, str, int, float, bool)):
+        return value
+
+    if hasattr(value, "model_dump"):
+        try:
+            return value.model_dump()
+        except Exception:  # noqa: BLE001
+            pass
+
+    if hasattr(value, "__dict__"):
+        try:
+            return {k: v for k, v in value.__dict__.items() if not k.startswith("_")}
+        except Exception:  # noqa: BLE001
+            pass
+
+    return value
+
+
+def _normalize_result_list(values: Any) -> List[Dict[str, Any]]:
+    """Normalize mixed SDK/list payloads into a list of dicts."""
+    if not isinstance(values, list):
+        return []
+
+    normalized: List[Dict[str, Any]] = []
+    for item in values:
+        plain = _to_plain_object(item)
+        if isinstance(plain, dict):
+            normalized.append(plain)
+    return normalized
+
+
+def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]:
+    """Extract Firecrawl search results across SDK/direct/gateway response shapes."""
+    response_plain = _to_plain_object(response)
+
+    if isinstance(response_plain, dict):
+        data = response_plain.get("data")
+        if isinstance(data, list):
+            return _normalize_result_list(data)
+
+        if isinstance(data, dict):
+            data_web = _normalize_result_list(data.get("web"))
+            if data_web:
+                return data_web
+            data_results = _normalize_result_list(data.get("results"))
+            if data_results:
+                return data_results
+
+        top_web = _normalize_result_list(response_plain.get("web"))
+        if top_web:
+            return top_web
+
+        top_results = _normalize_result_list(response_plain.get("results"))
+        if top_results:
+            return top_results
+
+    if hasattr(response, "web"):
+        return _normalize_result_list(getattr(response, "web", []))
+
+    return []
+
+
+def _extract_scrape_payload(scrape_result: Any) -> Dict[str, Any]:
+    """Normalize Firecrawl scrape payload shape across SDK and gateway variants."""
+    result_plain = _to_plain_object(scrape_result)
+    if not isinstance(result_plain, dict):
+        return {}
+
+    nested = result_plain.get("data")
+    if isinstance(nested, dict):
+        return nested
+
+    return result_plain
+
+
+# ---------------------------------------------------------------------------
+# Provider class
+# ---------------------------------------------------------------------------
+
+
+class FirecrawlWebSearchProvider(WebSearchProvider):
+    """Firecrawl search + extract provider with dual auth paths."""
+
+    @property
+    def name(self) -> str:
+        return "firecrawl"
+
+    @property
+    def display_name(self) -> str:
+        return "Firecrawl"
+
+    def is_available(self) -> bool:
+        """Return True when direct Firecrawl OR managed-gateway path is configured."""
+        return check_firecrawl_api_key()
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Firecrawl search.
+
+        Sync; matches the legacy ``_get_firecrawl_client().search(...)``
+        call directly. Normalizes the response across SDK/direct/gateway
+        shapes via :func:`_extract_web_search_results`.
+
+        Pre-flight errors (``ValueError`` from configuration check,
+        ``ImportError`` from missing SDK) propagate to the dispatcher's
+        top-level handler, which wraps them as ``tool_error(...)`` —
+        matching the legacy ``{"error": "Error searching web: ..."}``
+        envelope. Only in-flight errors are caught and surfaced as
+        ``{"success": False, "error": ...}``.
+        """
+        from tools.interrupt import is_interrupted
+
+        if is_interrupted():
+            return {"success": False, "error": "Interrupted"}
+
+        logger.info("Firecrawl search: '%s' (limit=%d)", query, limit)
+        # _get_firecrawl_client() raises ValueError on unconfigured systems —
+        # let it propagate so the dispatcher emits the legacy envelope shape.
+        client = _get_firecrawl_client()
+        try:
+            response = client.search(query=query, limit=limit)
+            web_results = _extract_web_search_results(response)
+            logger.info("Firecrawl: found %d search results", len(web_results))
+            return {"success": True, "data": {"web": web_results}}
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Firecrawl search error: %s", exc)
+            return {"success": False, "error": f"Firecrawl search failed: {exc}"}
+
+    async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via Firecrawl.
+
+        Async; each URL is scraped in a background thread with a 60s
+        timeout. After scraping, the final URL (post-redirect) is
+        re-checked against website-access policy.
+
+        Accepted kwargs (others ignored for forward compat):
+          - ``format``: ``"markdown"`` or ``"html"``; default is both
+            (request both, return markdown when available).
+
+        Returns the legacy per-URL list-of-results shape. Per-URL failures
+        (timeout, SSRF block, scrape error, policy block) become items
+        with an ``error`` field rather than raising.
+        """
+        from tools.interrupt import is_interrupted as _is_interrupted
+
+        if _is_interrupted():
+            return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
+
+        format = kwargs.get("format")
+        formats: List[str] = []
+        if format == "markdown":
+            formats = ["markdown"]
+        elif format == "html":
+            formats = ["html"]
+        else:
+            formats = ["markdown", "html"]
+
+        # check_website_access is the legacy policy gate; imported at
+        # module level (lazy-friendly because the website_policy import is
+        # cheap) so monkeypatching it in tests works as expected.
+
+        results: List[Dict[str, Any]] = []
+
+        for url in urls:
+            if _is_interrupted():
+                results.append({"url": url, "error": "Interrupted", "title": ""})
+                continue
+
+            # Pre-scrape website policy gate
+            blocked = check_website_access(url)
+            if blocked:
+                logger.info(
+                    "Blocked web_extract for %s by rule %s",
+                    blocked["host"],
+                    blocked["rule"],
+                )
+                results.append(
+                    {
+                        "url": url,
+                        "title": "",
+                        "content": "",
+                        "error": blocked["message"],
+                        "blocked_by_policy": {
+                            "host": blocked["host"],
+                            "rule": blocked["rule"],
+                            "source": blocked["source"],
+                        },
+                    }
+                )
+                continue
+
+            try:
+                logger.info("Firecrawl scraping: %s", url)
+                try:
+                    scrape_result = await asyncio.wait_for(
+                        asyncio.to_thread(
+                            _get_firecrawl_client().scrape,
+                            url=url,
+                            formats=formats,
+                        ),
+                        timeout=60,
+                    )
+                except asyncio.TimeoutError:
+                    logger.warning("Firecrawl scrape timed out for %s", url)
+                    results.append(
+                        {
+                            "url": url,
+                            "title": "",
+                            "content": "",
+                            "error": (
+                                "Scrape timed out after 60s — page may be too large "
+                                "or unresponsive. Try browser_navigate instead."
+                            ),
+                        }
+                    )
+                    continue
+
+                scrape_payload = _extract_scrape_payload(scrape_result)
+                metadata = scrape_payload.get("metadata", {})
+                content_markdown = scrape_payload.get("markdown")
+                content_html = scrape_payload.get("html")
+
+                # Ensure metadata is a dict (SDK may return a typed object)
+                if not isinstance(metadata, dict):
+                    if hasattr(metadata, "model_dump"):
+                        metadata = metadata.model_dump()
+                    elif hasattr(metadata, "__dict__"):
+                        metadata = metadata.__dict__
+                    else:
+                        metadata = {}
+
+                title = metadata.get("title", "")
+                final_url = metadata.get("sourceURL", url)
+
+                # Re-check website-access policy after any redirect
+                final_blocked = check_website_access(final_url)
+                if final_blocked:
+                    logger.info(
+                        "Blocked redirected web_extract for %s by rule %s",
+                        final_blocked["host"],
+                        final_blocked["rule"],
+                    )
+                    results.append(
+                        {
+                            "url": final_url,
+                            "title": title,
+                            "content": "",
+                            "raw_content": "",
+                            "error": final_blocked["message"],
+                            "blocked_by_policy": {
+                                "host": final_blocked["host"],
+                                "rule": final_blocked["rule"],
+                                "source": final_blocked["source"],
+                            },
+                        }
+                    )
+                    continue
+
+                # Choose markdown vs html according to the requested format
+                if format == "markdown" or (format is None and content_markdown):
+                    chosen_content = content_markdown
+                else:
+                    chosen_content = content_html or content_markdown or ""
+
+                results.append(
+                    {
+                        "url": final_url,
+                        "title": title,
+                        "content": chosen_content,
+                        "raw_content": chosen_content,
+                        "metadata": metadata,
+                    }
+                )
+            except Exception as scrape_err:  # noqa: BLE001
+                logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
+                results.append(
+                    {
+                        "url": url,
+                        "title": "",
+                        "content": "",
+                        "raw_content": "",
+                        "error": str(scrape_err),
+                    }
+                )
+
+        return results
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Firecrawl",
+            "badge": "paid · optional gateway",
+            "tag": (
+                "Full search + extract; supports direct API and "
+                "Nous tool-gateway routing."
+            ),
+            "env_vars": [
+                {
+                    "key": "FIRECRAWL_API_KEY",
+                    "prompt": "Firecrawl API key (or leave blank for self-hosted)",
+                    "url": "https://docs.firecrawl.dev/introduction",
+                },
+            ],
+        }

plugins/web/parallel/__init__.py

@@ -0,0 +1,16 @@
+"""Parallel.ai web search + extract plugin — bundled, auto-loaded.
+
+First plugin in this repo to expose an async :meth:`extract` — Parallel's
+SDK is async-native (``AsyncParallel.beta.extract``). The web_extract_tool
+dispatcher detects coroutines via :func:`inspect.iscoroutinefunction` and
+awaits.
+"""
+
+from __future__ import annotations
+
+from plugins.web.parallel.provider import ParallelWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Parallel provider with the plugin context."""
+    ctx.register_web_search_provider(ParallelWebSearchProvider())

plugins/web/parallel/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-parallel
+version: 1.0.0
+description: "Parallel.ai web search + content extraction. Search returns objective-tuned results; extract uses the async SDK for parallel page fetches. Requires PARALLEL_API_KEY — sign up at https://parallel.ai."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - parallel

plugins/web/parallel/provider.py

@@ -0,0 +1,696 @@
+"""Parallel.ai web search + content extraction — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`.
+
+Search runs on one of two transports, picked by credential:
+
+- **No key →** the free hosted Search MCP at ``https://search.parallel.ai/mcp``
+  (anonymous Streamable-HTTP JSON-RPC). This makes ``web_search`` work out of
+  the box with zero setup, which is why ``parallel`` is the keyless default
+  backend in :func:`tools.web_tools._get_backend`.
+- **``PARALLEL_API_KEY`` →** the ``parallel`` SDK's v1 ``search`` / ``extract``
+  REST endpoints (objective-tuned, mode-selectable, higher rate limits).
+
+Extract mirrors search: keyed uses the async SDK (``AsyncParallel``) v1
+``extract``; keyless uses the free MCP's ``web_fetch``. :meth:`extract` is
+declared ``async def`` and the dispatcher in
+:func:`tools.web_tools.web_extract_tool` detects coroutines via
+:func:`inspect.iscoroutinefunction` and awaits.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "parallel"      # explicit per-capability
+      extract_backend: "parallel"     # explicit per-capability
+      backend: "parallel"             # shared fallback
+      # Optional: search mode (default "advanced"; also "basic")
+      # via the PARALLEL_SEARCH_MODE env var. REST path only.
+
+Env vars::
+
+    PARALLEL_API_KEY=...             # https://parallel.ai (optional — unlocks
+                                     # the v1 REST Search API; without it,
+                                     # search and extract use the free MCP)
+    PARALLEL_SEARCH_MODE=advanced    # optional: basic|advanced (legacy
+                                     # fast/one-shot map to basic, agentic to
+                                     # advanced). REST path only.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import uuid
+from typing import Any, Dict, List
+
+import httpx
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+# Free hosted Search MCP — anonymous-friendly, used when no PARALLEL_API_KEY is
+# configured. Docs: https://docs.parallel.ai/integrations/mcp/search-mcp
+_MCP_SEARCH_URL = "https://search.parallel.ai/mcp"
+_MCP_PROTOCOL_VERSION = "2025-06-18"
+# Deliberately generic client identity. Project policy (see the telemetry PR
+# policy in AGENTS.md) forbids third-party usage attribution without an
+# explicit user opt-in, so neither clientInfo nor the User-Agent names
+# hermes. MCP requires *a* clientInfo; a neutral one satisfies the spec
+# without attributing traffic.
+_MCP_CLIENT_NAME = "mcp-web-client"
+_MCP_CLIENT_VERSION = "1.0.0"
+_MCP_USER_AGENT = f"{_MCP_CLIENT_NAME}/{_MCP_CLIENT_VERSION}"
+_MCP_TIMEOUT_SECONDS = 30.0
+
+# Free-tier attribution. The hosted Search MCP is free to use; surfacing this
+# on keyless results credits Parallel and matches the free-tier terms
+# (https://parallel.ai/customer-terms).
+_FREE_MCP_ATTRIBUTION = (
+    "Search powered by the free Parallel Web Search MCP (https://parallel.ai)."
+)
+
+
+def _new_session_id() -> str:
+    """Mint a fresh Parallel ``session_id`` for a single tool call.
+
+    Per-call rather than process-global: one process serves many unrelated
+    chats in the gateway/batch runners, and a shared id would pool their
+    searches into one Parallel session. The prefix is deliberately generic
+    (no hermes attribution — telemetry policy).
+    """
+    return f"{_MCP_CLIENT_NAME}-{uuid.uuid4().hex}"
+
+# Module-level note: the canonical cache slots ``_parallel_client`` and
+# ``_async_parallel_client`` live on :mod:`tools.web_tools` so tests that do
+# ``tools.web_tools._parallel_client = None`` between cases see fresh state.
+# The plugin reads/writes through that public module (see
+# :func:`_get_sync_client` / :func:`_get_async_client`).
+
+
+def _ensure_parallel_sdk_installed() -> None:
+    """Trigger lazy install of the parallel SDK if it isn't present.
+
+    Mirrors the lazy-deps pattern used by the legacy implementation.
+    Swallows benign ImportError from the lazy_deps helper itself; if the
+    SDK is genuinely missing the subsequent ``from parallel import ...``
+    raises ImportError that the caller can handle.
+    """
+    try:
+        from tools.lazy_deps import ensure as _lazy_ensure
+
+        _lazy_ensure("search.parallel", prompt=False)
+    except ImportError:
+        pass
+    except Exception as exc:  # noqa: BLE001 — surface install hint as ImportError
+        raise ImportError(str(exc))
+
+
+def _get_sync_client() -> Any:
+    """Lazy-load + cache the sync Parallel client.
+
+    Cache lives on :mod:`tools.web_tools` (as ``_parallel_client``) so unit
+    tests that reset that name between cases keep working.
+    """
+    import tools.web_tools as _wt
+
+    cached = getattr(_wt, "_parallel_client", None)
+    if cached is not None:
+        return cached
+
+    api_key = os.getenv("PARALLEL_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "PARALLEL_API_KEY environment variable not set. "
+            "Get your API key at https://parallel.ai"
+        )
+
+    _ensure_parallel_sdk_installed()
+    from parallel import Parallel  # noqa: WPS433 — deliberately lazy
+
+    client = Parallel(api_key=api_key)
+    _wt._parallel_client = client
+    return client
+
+
+def _get_async_client() -> Any:
+    """Lazy-load + cache the async Parallel client.
+
+    Cache lives on :mod:`tools.web_tools` (as ``_async_parallel_client``).
+    """
+    import tools.web_tools as _wt
+
+    cached = getattr(_wt, "_async_parallel_client", None)
+    if cached is not None:
+        return cached
+
+    api_key = os.getenv("PARALLEL_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "PARALLEL_API_KEY environment variable not set. "
+            "Get your API key at https://parallel.ai"
+        )
+
+    _ensure_parallel_sdk_installed()
+    from parallel import AsyncParallel  # noqa: WPS433 — deliberately lazy
+
+    client = AsyncParallel(api_key=api_key)
+    _wt._async_parallel_client = client
+    return client
+
+
+def _reset_clients_for_tests() -> None:
+    """Drop both cached clients so tests can re-instantiate cleanly.
+
+    Clears the canonical slots on :mod:`tools.web_tools` (where
+    :func:`_get_sync_client` / :func:`_get_async_client` read/write them).
+    """
+    import tools.web_tools as _wt
+
+    _wt._parallel_client = None
+    _wt._async_parallel_client = None
+
+
+# Backward-compatible aliases for the names that lived in tools.web_tools
+# before the migration (matches existing tests + external callers).
+_get_parallel_client = _get_sync_client
+_get_async_parallel_client = _get_async_client
+
+
+def _resolve_search_mode() -> str:
+    """Return the validated v1 search mode (default "advanced").
+
+    V1 collapses the three Beta modes into two. We accept the v1 values
+    directly and map the legacy Beta values for back-compat with anyone who
+    still sets ``PARALLEL_SEARCH_MODE=fast|one-shot|agentic``:
+
+    - ``fast`` / ``one-shot`` → ``basic``  (lower latency)
+    - ``agentic``             → ``advanced`` (higher quality, the v1 default)
+    """
+    mode = os.getenv("PARALLEL_SEARCH_MODE", "advanced").lower().strip()
+    if mode == "basic" or mode in {"fast", "one-shot"}:
+        return "basic"
+    # advanced, legacy "agentic", and anything unrecognized → the v1 default.
+    return "advanced"
+
+
+# ---------------------------------------------------------------------------
+# Free Search MCP transport (keyless path)
+# ---------------------------------------------------------------------------
+#
+# A small hand-rolled Streamable-HTTP JSON-RPC client for the hosted Search
+# MCP, rather than the full MCP-client subsystem: we only call two tools
+# (``web_search`` / ``web_fetch``), so keeping it inline lets web_search and
+# web_extract stay ordinary tools with the MCP endpoint as just their wire
+# protocol.
+
+
+def _mcp_headers(
+    session_id: str | None,
+    api_key: str | None,
+    protocol_version: str | None = None,
+) -> Dict[str, str]:
+    """Headers for an MCP request.
+
+    A Bearer token is attached only when we actually hold a key — the free
+    endpoint is anonymous, and sending an empty/garbage token would make it
+    401 instead of serving the anonymous tier. After ``initialize`` the
+    Streamable-HTTP spec expects the negotiated ``MCP-Protocol-Version`` on
+    every follow-up request, so we echo it once known.
+    """
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json, text/event-stream",
+        "User-Agent": _MCP_USER_AGENT,
+    }
+    if session_id:
+        headers["Mcp-Session-Id"] = session_id
+    if protocol_version:
+        headers["MCP-Protocol-Version"] = protocol_version
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    return headers
+
+
+def _iter_mcp_messages(text: str):
+    """Yield JSON-RPC message dicts from a plain-JSON or SSE response body.
+
+    Handles ``application/json`` (a single object) and ``text/event-stream``
+    (SSE: events separated by blank lines; an event's one-or-more ``data:``
+    lines concatenate into a single JSON payload). Unparseable chunks and
+    non-``data`` SSE fields (``event:``/``id:``/comments) are skipped.
+    """
+    def _emit(payload):
+        # Streamable HTTP allows batching responses/notifications into a JSON
+        # array — flatten so callers always see individual message dicts.
+        if isinstance(payload, list):
+            yield from payload
+        elif payload is not None:
+            yield payload
+
+    body = (text or "").strip()
+    if not body:
+        return
+    if body.startswith("{") or body.startswith("["):
+        try:
+            parsed = json.loads(body)
+        except json.JSONDecodeError:
+            return
+        yield from _emit(parsed)
+        return
+
+    data_lines: List[str] = []
+
+    def _flush():
+        if not data_lines:
+            return None
+        try:
+            return json.loads("\n".join(data_lines))
+        except json.JSONDecodeError:
+            return None
+
+    for raw in body.split("\n"):
+        line = raw.rstrip("\r")
+        if line.startswith("data:"):
+            data_lines.append(line[len("data:"):].lstrip())
+        elif line.strip() == "":  # event boundary
+            yield from _emit(_flush())
+            data_lines = []
+    yield from _emit(_flush())
+
+
+def _mcp_response_envelope(text: str, request_id: str) -> Dict[str, Any]:
+    """Select the JSON-RPC response for *request_id* from an MCP response body.
+
+    Streamable-HTTP servers may emit progress/log notifications before the
+    final result, so we scan the whole stream and return the result/error
+    message whose ``id`` matches our request. Falls back to the last
+    result/error-bearing message if no id matches; ``{}`` if none is present.
+    """
+    fallback: Dict[str, Any] = {}
+    for msg in _iter_mcp_messages(text):
+        if not isinstance(msg, dict) or not ("result" in msg or "error" in msg):
+            continue
+        if msg.get("id") == request_id:
+            return msg
+        fallback = msg
+    return fallback
+
+
+def _mcp_payload(envelope: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract the tool result payload from a ``tools/call`` envelope.
+
+    Prefers ``structuredContent`` (authoritative machine-readable form);
+    otherwise scans text blocks for the first JSON-parseable one. Raises on a
+    JSON-RPC error or a tool-level ``isError``.
+    """
+    if "error" in envelope:
+        raise RuntimeError(f"Parallel MCP error: {str(envelope['error'])[:500]}")
+    result = envelope.get("result") or {}
+    if result.get("isError"):
+        raise RuntimeError(f"Parallel MCP tool error: {str(result)[:500]}")
+
+    structured = result.get("structuredContent")
+    if isinstance(structured, dict):
+        return structured
+
+    for block in result.get("content", []) or []:
+        if isinstance(block, dict) and block.get("type") == "text":
+            text = str(block.get("text") or "")
+            if not text:
+                continue
+            try:
+                return json.loads(text)
+            except json.JSONDecodeError:
+                continue
+    raise RuntimeError(
+        f"Parallel MCP returned no parseable content: {str(result)[:500]}"
+    )
+
+
+def _mcp_call(
+    tool_name: str, arguments: Dict[str, Any], api_key: str | None
+) -> Dict[str, Any]:
+    """Run the MCP handshake then a single ``tools/call`` and return its payload.
+
+    initialize → (capture ``Mcp-Session-Id``) → notifications/initialized →
+    tools/call ``tool_name``. Returns the parsed tool payload dict (see
+    :func:`_mcp_payload`). A Bearer token is attached only when *api_key* is set.
+    """
+    with httpx.Client(timeout=_MCP_TIMEOUT_SECONDS) as client:
+        # 1. initialize — capture the server-assigned MCP session id.
+        init_id = str(uuid.uuid4())
+        init = client.post(
+            _MCP_SEARCH_URL,
+            headers=_mcp_headers(None, api_key),
+            json={
+                "jsonrpc": "2.0",
+                "id": init_id,
+                "method": "initialize",
+                "params": {
+                    "protocolVersion": _MCP_PROTOCOL_VERSION,
+                    "capabilities": {},
+                    "clientInfo": {
+                        "name": _MCP_CLIENT_NAME,
+                        "version": _MCP_CLIENT_VERSION,
+                    },
+                },
+            },
+        )
+        init.raise_for_status()
+        # Only echo a session id the server actually issued. Stateless
+        # Streamable-HTTP servers may omit it; inventing one and sending it on
+        # follow-up requests can get those requests rejected (the server never
+        # created that session). When absent, the Mcp-Session-Id header is simply
+        # omitted (see _mcp_headers). This is separate from the tool-arg
+        # ``session_id`` below, which is a client-minted rate-limit/grouping id.
+        mcp_session_id = init.headers.get("mcp-session-id")
+        init_env = _mcp_response_envelope(init.text, init_id)
+        # Echo the negotiated protocol version on every post-init request, per
+        # the Streamable-HTTP spec (servers may enforce it).
+        negotiated_version = (
+            (init_env.get("result") or {}).get("protocolVersion")
+            or _MCP_PROTOCOL_VERSION
+        )
+
+        # 2. notifications/initialized — required handshake ack.
+        client.post(
+            _MCP_SEARCH_URL,
+            headers=_mcp_headers(mcp_session_id, api_key, negotiated_version),
+            json={"jsonrpc": "2.0", "method": "notifications/initialized"},
+        )
+
+        # 3. tools/call.
+        call_id = str(uuid.uuid4())
+        call = client.post(
+            _MCP_SEARCH_URL,
+            headers=_mcp_headers(mcp_session_id, api_key, negotiated_version),
+            json={
+                "jsonrpc": "2.0",
+                "id": call_id,
+                "method": "tools/call",
+                "params": {"name": tool_name, "arguments": arguments},
+            },
+        )
+        call.raise_for_status()
+        return _mcp_payload(_mcp_response_envelope(call.text, call_id))
+
+
+def _mcp_web_search(query: str, limit: int, api_key: str | None) -> Dict[str, Any]:
+    """Run a ``web_search`` tool call against the hosted Search MCP.
+
+    Returns the standard provider search shape
+    (``{"success": True, "data": {"web": [...]}}``). The MCP serves a fixed
+    result count, so ``limit`` is applied client-side. The MCP requires
+    ``objective`` (REST treats it as optional), so we mirror the query.
+    """
+    payload = _mcp_call(
+        "web_search",
+        {
+            "objective": query,
+            "search_queries": [query],
+            "session_id": _new_session_id(),
+        },
+        api_key,
+    )
+
+    web_results: List[Dict[str, Any]] = []
+    for i, result in enumerate((payload.get("results") or [])[: max(limit, 1)]):
+        if not isinstance(result, dict):
+            continue
+        excerpts = result.get("excerpts") or []
+        web_results.append(
+            {
+                "url": result.get("url") or "",
+                "title": result.get("title") or "",
+                "description": " ".join(excerpts) if excerpts else "",
+                "position": i + 1,
+            }
+        )
+
+    # Credit the free tier (anonymous path only — keyed search uses REST and
+    # carries no attribution).
+    return {
+        "success": True,
+        "data": {"web": web_results},
+        "provider": "parallel",
+        "attribution": _FREE_MCP_ATTRIBUTION,
+    }
+
+
+def _mcp_web_fetch(urls: List[str], api_key: str | None) -> List[Dict[str, Any]]:
+    """Run a ``web_fetch`` tool call against the hosted Search MCP.
+
+    Returns the per-URL extract shape that
+    :func:`tools.web_tools.web_extract_tool` expects — exactly one row per input
+    URL, in request order (including duplicates). We pass ``full_content=True``
+    so the page body comes back as markdown (matching the keyed SDK path and
+    what extract callers/summarizers expect), falling back to excerpts only when
+    full content is absent. Any input the MCP didn't return is emitted as a
+    per-URL error row.
+    """
+    payload = _mcp_call(
+        "web_fetch",
+        {"urls": list(urls), "full_content": True, "session_id": _new_session_id()},
+        api_key,
+    )
+
+    # Index the response by URL, then emit one row per *input* URL in order so
+    # duplicates and positional alignment with the request list are preserved.
+    by_url: Dict[str, Dict[str, Any]] = {}
+    for item in payload.get("results") or []:
+        if isinstance(item, dict) and item.get("url"):
+            by_url.setdefault(item["url"], item)
+
+    results: List[Dict[str, Any]] = []
+    for url in urls:
+        item = by_url.get(url)
+        if item is None:
+            results.append(
+                {
+                    "url": url,
+                    "title": "",
+                    "content": "",
+                    "error": "extraction failed (no content returned)",
+                    "metadata": {"sourceURL": url},
+                }
+            )
+            continue
+        title = item.get("title") or ""
+        # Prefer the full page body; fall back to joined excerpts (mirrors the
+        # keyed SDK extract path).
+        content = item.get("full_content") or "\n\n".join(item.get("excerpts") or [])
+        results.append(
+            {
+                "url": url,
+                "title": title,
+                "content": content,
+                "raw_content": content,
+                "metadata": {"sourceURL": url, "title": title},
+            }
+        )
+
+    return results
+
+
+class ParallelWebSearchProvider(WebSearchProvider):
+    """Parallel.ai search + async extract provider."""
+
+    @property
+    def name(self) -> str:
+        return "parallel"
+
+    @property
+    def display_name(self) -> str:
+        return "Parallel"
+
+    def is_available(self) -> bool:
+        """Return True when ``PARALLEL_API_KEY`` is set.
+
+        Deliberately key-based: this gates the registry's active-provider walk
+        and the ``hermes tools`` picker (auto-selecting Parallel for a user who
+        hasn't named it), so it must not claim availability on the keyless path.
+        The keyless free-MCP path is reached independently via
+        :func:`tools.web_tools._get_backend`'s ``parallel`` terminal default.
+        """
+        return bool(os.getenv("PARALLEL_API_KEY", "").strip())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Parallel search (sync).
+
+        With ``PARALLEL_API_KEY`` set, uses the v1 ``search`` REST endpoint with
+        the configured mode (``PARALLEL_SEARCH_MODE`` env var, default
+        "advanced"; limit requested via advanced_settings.max_results, capped at
+        20). Without a key, falls back to the free hosted Search MCP so search
+        still works with zero setup.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            api_key = os.getenv("PARALLEL_API_KEY", "").strip()
+            if not api_key:
+                logger.info(
+                    "Parallel search (free MCP): '%s' (limit=%d)", query, limit
+                )
+                return _mcp_web_search(query, limit, api_key=None)
+
+            mode = _resolve_search_mode()
+            logger.info(
+                "Parallel search (v1 REST): '%s' (mode=%s, limit=%d)",
+                query, mode, limit,
+            )
+            # v1 Search API. Request the caller's limit via max_results (capped
+            # at 20) so we don't rely on the API default — the slice below can
+            # only trim, not ask for more.
+            response = _get_sync_client().search(
+                search_queries=[query],
+                objective=query,
+                mode=mode,
+                session_id=_new_session_id(),
+                advanced_settings={"max_results": min(max(limit, 1), 20)},
+            )
+
+            web_results = []
+            for i, result in enumerate((response.results or [])[: max(limit, 1)]):
+                excerpts = result.excerpts or []
+                web_results.append(
+                    {
+                        "url": result.url or "",
+                        "title": result.title or "",
+                        "description": " ".join(excerpts) if excerpts else "",
+                        "position": i + 1,
+                    }
+                )
+
+            # Paid/REST path: no attribution and no "[Parallel]" label — the
+            # branding is specifically for the free Search MCP tier.
+            return {"success": True, "data": {"web": web_results}}
+        except ValueError as exc:
+            return {"success": False, "error": str(exc)}
+        except ImportError as exc:
+            return {
+                "success": False,
+                "error": f"Parallel SDK not installed: {exc}",
+            }
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Parallel search error: %s", exc)
+            return {"success": False, "error": f"Parallel search failed: {exc}"}
+
+    async def extract(
+        self, urls: List[str], **kwargs: Any
+    ) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs.
+
+        With ``PARALLEL_API_KEY`` set, uses the async SDK's v1 ``extract`` for
+        full page content. Without a key, falls back to the free hosted Search
+        MCP's ``web_fetch`` tool so extraction works with zero setup, mirroring
+        the keyless search path.
+
+        Returns the legacy list-of-results shape that
+        :func:`tools.web_tools.web_extract_tool` expects: one entry per
+        successful URL plus one entry per failed URL with an ``error``
+        field. Errors are not raised — they're returned as per-URL items.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            api_key = os.getenv("PARALLEL_API_KEY", "").strip()
+            if not api_key:
+                logger.info(
+                    "Parallel extract (free MCP web_fetch): %d URL(s)", len(urls)
+                )
+                # _mcp_web_fetch is sync httpx; run off the event loop.
+                return await asyncio.to_thread(_mcp_web_fetch, list(urls), None)
+
+            logger.info("Parallel extract (v1 REST): %d URL(s)", len(urls))
+            # v1 Extract API (client.extract, /v1/extract); full_content is set
+            # via advanced_settings.
+            response = await _get_async_client().extract(
+                urls=urls,
+                advanced_settings={"full_content": True},
+                session_id=_new_session_id(),
+            )
+
+            results: List[Dict[str, Any]] = []
+            for result in response.results or []:
+                content = result.full_content or ""
+                if not content:
+                    content = "\n\n".join(result.excerpts or [])
+                url = result.url or ""
+                title = result.title or ""
+                results.append(
+                    {
+                        "url": url,
+                        "title": title,
+                        "content": content,
+                        "raw_content": content,
+                        "metadata": {"sourceURL": url, "title": title},
+                    }
+                )
+
+            for error in response.errors or []:
+                err_url = getattr(error, "url", "") or ""
+                err_msg = (
+                    getattr(error, "message", None)
+                    or getattr(error, "content", None)
+                    or getattr(error, "error_type", None)
+                    or "extraction failed"
+                )
+                results.append(
+                    {
+                        "url": err_url,
+                        "title": "",
+                        "content": "",
+                        "error": err_msg,
+                        "metadata": {"sourceURL": err_url},
+                    }
+                )
+
+            return results
+        except ValueError as exc:
+            return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
+        except ImportError as exc:
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Parallel SDK not installed: {exc}"}
+                for u in urls
+            ]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Parallel extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Parallel extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Parallel",
+            "badge": "free",
+            "tag": (
+                "Free web search + extraction via Parallel's hosted Search MCP "
+                "— no key needed. Add PARALLEL_API_KEY for the v1 REST Search "
+                "API (richer modes, higher limits)."
+            ),
+            "env_vars": [
+                {
+                    "key": "PARALLEL_API_KEY",
+                    "prompt": "Parallel API key (optional — unlocks the v1 REST Search API)",
+                    "url": "https://parallel.ai",
+                },
+            ],
+        }

plugins/web/searxng/__init__.py

@@ -0,0 +1,15 @@
+"""SearXNG search plugin — bundled, auto-loaded.
+
+Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``).
+Search-only — pair with an extract provider (firecrawl/tavily/exa) for
+``web_extract`` calls.
+"""
+
+from __future__ import annotations
+
+from plugins.web.searxng.provider import SearXNGWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the SearXNG provider with the plugin context."""
+    ctx.register_web_search_provider(SearXNGWebSearchProvider())

plugins/web/searxng/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-searxng
+version: 1.0.0
+description: "SearXNG web search — free, self-hosted, privacy-respecting metasearch engine. Requires SEARXNG_URL pointing at your instance."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - searxng

plugins/web/searxng/provider.py

@@ -0,0 +1,153 @@
+"""SearXNG search — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Same JSON
+API call (``/search?format=json``), same result normalization. The legacy
+in-tree module ``tools.web_providers.searxng`` was removed in the same
+commit that moved this code under ``plugins/``; this file is now the
+canonical implementation.
+
+Search-only — SearXNG aggregates results from upstream engines but does not
+fetch/extract arbitrary URLs. ``supports_extract()`` returns False.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "searxng"     # explicit per-capability
+      backend: "searxng"            # shared fallback
+
+Env var::
+
+    SEARXNG_URL=http://localhost:8080
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+
+def _searxng_url() -> str:
+    """Return SEARXNG_URL from Hermes config-aware env, falling back to process env."""
+    try:
+        from hermes_cli.config import get_env_value
+
+        val = get_env_value("SEARXNG_URL")
+    except Exception:
+        val = None
+    if val is None:
+        val = os.getenv("SEARXNG_URL", "")
+    return (val or "").strip()
+
+
+class SearXNGWebSearchProvider(WebSearchProvider):
+    """Search via a user-hosted SearXNG instance."""
+
+    @property
+    def name(self) -> str:
+        return "searxng"
+
+    @property
+    def display_name(self) -> str:
+        return "SearXNG"
+
+    def is_available(self) -> bool:
+        """Return True when ``SEARXNG_URL`` is set."""
+        return bool(_searxng_url())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a search against the configured SearXNG instance."""
+        import httpx
+
+        base_url = _searxng_url().rstrip("/")
+        if not base_url:
+            return {"success": False, "error": "SEARXNG_URL is not set"}
+
+        params: Dict[str, Any] = {
+            "q": query,
+            "format": "json",
+            "pageno": 1,
+        }
+
+        try:
+            resp = httpx.get(
+                f"{base_url}/search",
+                params=params,
+                timeout=15,
+                headers={"Accept": "application/json"},
+            )
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            logger.warning("SearXNG HTTP error: %s", exc)
+            return {
+                "success": False,
+                "error": f"SearXNG returned HTTP {exc.response.status_code}",
+            }
+        except httpx.RequestError as exc:
+            logger.warning("SearXNG request error: %s", exc)
+            return {
+                "success": False,
+                "error": f"Could not reach SearXNG at {base_url}: {exc}",
+            }
+
+        try:
+            data = resp.json()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("SearXNG response parse error: %s", exc)
+            return {
+                "success": False,
+                "error": "Could not parse SearXNG response as JSON",
+            }
+
+        raw_results = data.get("results", [])
+
+        # SearXNG may return a score field; sort descending and cap to limit.
+        sorted_results = sorted(
+            raw_results,
+            key=lambda r: float(r.get("score", 0)),
+            reverse=True,
+        )[:limit]
+
+        web_results = [
+            {
+                "title": str(r.get("title", "")),
+                "url": str(r.get("url", "")),
+                "description": str(r.get("content", "")),
+                "position": i + 1,
+            }
+            for i, r in enumerate(sorted_results)
+        ]
+
+        logger.info(
+            "SearXNG search '%s': %d results (from %d raw, limit %d)",
+            query,
+            len(web_results),
+            len(raw_results),
+            limit,
+        )
+
+        return {"success": True, "data": {"web": web_results}}
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "SearXNG",
+            "badge": "free · self-hosted",
+            "tag": "Free, privacy-respecting metasearch. Point SEARXNG_URL at your instance.",
+            "env_vars": [
+                {
+                    "key": "SEARXNG_URL",
+                    "prompt": "SearXNG instance URL (e.g. http://localhost:8080)",
+                    "url": "https://searx.space/",
+                },
+            ],
+        }

plugins/web/tavily/__init__.py

@@ -0,0 +1,10 @@
+"""Tavily web search + extract plugin — bundled, auto-loaded."""
+
+from __future__ import annotations
+
+from plugins.web.tavily.provider import TavilyWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Tavily provider with the plugin context."""
+    ctx.register_web_search_provider(TavilyWebSearchProvider())

plugins/web/tavily/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-tavily
+version: 1.0.0
+description: "Tavily web search + content extraction + crawl. Search + extract are mainstream; crawl is unique to Tavily among built-in providers. Requires TAVILY_API_KEY — sign up at https://app.tavily.com/home."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - tavily

plugins/web/tavily/provider.py

@@ -0,0 +1,220 @@
+"""Tavily web search + content extraction — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two
+capabilities advertised:
+
+- ``supports_search()``  -> True (Tavily ``/search``)
+- ``supports_extract()`` -> True (Tavily ``/extract``)
+
+Both are sync — the underlying call is ``httpx.post(...)``.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "tavily"     # explicit per-capability
+      extract_backend: "tavily"    # explicit per-capability
+      backend: "tavily"            # shared fallback for both
+
+Env vars::
+
+    TAVILY_API_KEY=...           # https://app.tavily.com/home (required)
+    TAVILY_BASE_URL=...          # optional override of https://api.tavily.com
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, List
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+
+def _tavily_request(endpoint: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+    """POST to the Tavily API and return the parsed JSON response.
+
+    Mirrors :func:`tools.web_tools._tavily_request`. Raises ``ValueError``
+    when ``TAVILY_API_KEY`` is unset; the caller catches and surfaces as
+    a typed error response.
+    """
+    import httpx
+
+    api_key = os.getenv("TAVILY_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "TAVILY_API_KEY environment variable not set. "
+            "Get your API key at https://app.tavily.com/home"
+        )
+
+    base_url = os.getenv("TAVILY_BASE_URL", "https://api.tavily.com")
+    payload = dict(payload)  # don't mutate caller's dict
+    payload["api_key"] = api_key
+    url = f"{base_url}/{endpoint.lstrip('/')}"
+    logger.info("Tavily %s request to %s", endpoint, url)
+
+    response = httpx.post(url, json=payload, timeout=60)
+    response.raise_for_status()
+    return response.json()
+
+
+def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]:
+    """Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``."""
+    web_results = []
+    for i, result in enumerate(response.get("results", [])):
+        web_results.append(
+            {
+                "title": result.get("title", ""),
+                "url": result.get("url", ""),
+                "description": result.get("content", ""),
+                "position": i + 1,
+            }
+        )
+    return {"success": True, "data": {"web": web_results}}
+
+
+def _normalize_tavily_documents(
+    response: Dict[str, Any], fallback_url: str = ""
+) -> List[Dict[str, Any]]:
+    """Map Tavily ``/extract`` response to standard documents.
+
+    Documents follow the legacy LLM post-processing shape::
+
+        {"url", "title", "content", "raw_content", "metadata"}
+
+    Failures (``failed_results``, ``failed_urls``) become result entries
+    with an ``error`` field rather than raising.
+    """
+    documents: List[Dict[str, Any]] = []
+    for result in response.get("results", []):
+        url = result.get("url", fallback_url)
+        raw = result.get("raw_content", "") or result.get("content", "")
+        documents.append(
+            {
+                "url": url,
+                "title": result.get("title", ""),
+                "content": raw,
+                "raw_content": raw,
+                "metadata": {"sourceURL": url, "title": result.get("title", "")},
+            }
+        )
+    for fail in response.get("failed_results", []):
+        documents.append(
+            {
+                "url": fail.get("url", fallback_url),
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": fail.get("error", "extraction failed"),
+                "metadata": {"sourceURL": fail.get("url", fallback_url)},
+            }
+        )
+    for fail_url in response.get("failed_urls", []):
+        url_str = fail_url if isinstance(fail_url, str) else str(fail_url)
+        documents.append(
+            {
+                "url": url_str,
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": "extraction failed",
+                "metadata": {"sourceURL": url_str},
+            }
+        )
+    return documents
+
+
+class TavilyWebSearchProvider(WebSearchProvider):
+    """Tavily search + extract provider."""
+
+    @property
+    def name(self) -> str:
+        return "tavily"
+
+    @property
+    def display_name(self) -> str:
+        return "Tavily"
+
+    def is_available(self) -> bool:
+        """Return True when ``TAVILY_API_KEY`` is set to a non-empty value."""
+        return bool(os.getenv("TAVILY_API_KEY", "").strip())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Tavily search."""
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            logger.info("Tavily search: '%s' (limit=%d)", query, limit)
+            raw = _tavily_request(
+                "search",
+                {
+                    "query": query,
+                    "max_results": min(limit, 20),
+                    "include_raw_content": False,
+                    "include_images": False,
+                },
+            )
+            return _normalize_tavily_search_results(raw)
+        except ValueError as exc:
+            return {"success": False, "error": str(exc)}
+        except Exception as exc:  # noqa: BLE001 — including httpx errors
+            logger.warning("Tavily search error: %s", exc)
+            return {"success": False, "error": f"Tavily search failed: {exc}"}
+
+    def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via Tavily.
+
+        Sync — the underlying call is httpx.post(...). Returns the legacy
+        list-of-results shape; per-URL failures become items with ``error``.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            logger.info("Tavily extract: %d URL(s)", len(urls))
+            raw = _tavily_request(
+                "extract",
+                {
+                    "urls": urls,
+                    "include_images": False,
+                },
+            )
+            return _normalize_tavily_documents(
+                raw, fallback_url=urls[0] if urls else ""
+            )
+        except ValueError as exc:
+            return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Tavily extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Tavily",
+            "badge": "paid",
+            "tag": "Search + extract in one provider.",
+            "env_vars": [
+                {
+                    "key": "TAVILY_API_KEY",
+                    "prompt": "Tavily API key",
+                    "url": "https://app.tavily.com/home",
+                },
+            ],
+        }

plugins/web/xai/__init__.py

@@ -0,0 +1,14 @@
+"""xAI web search plugin — bundled, auto-loaded.
+
+Mirrors the ``plugins/web/brave_free/`` layout: ``provider.py`` holds the
+provider class, ``__init__.py::register(ctx)`` registers an instance.
+"""
+
+from __future__ import annotations
+
+from plugins.web.xai.provider import XAIWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the xAI Web Search provider with the plugin context."""
+    ctx.register_web_search_provider(XAIWebSearchProvider())

plugins/web/xai/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-xai
+version: 1.0.0
+description: "xAI Web Search — search the web via Grok's agentic web_search tool (Responses API). Requires xAI Grok OAuth (via `hermes auth`) or XAI_API_KEY (https://x.ai)."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - xai

plugins/web/xai/provider.py

@@ -0,0 +1,557 @@
+"""xAI Web Search — plugin form.
+
+Routes ``web_search`` tool calls through xAI's agentic Web Search tool
+(server-side ``web_search`` on the Responses API). Grok runs the actual
+searching and page-browsing server-side; we ask it to return the top
+results as structured JSON so we can hand back the same
+``{title, url, description, position}`` rows every other Hermes web
+provider produces.
+
+Reference: https://docs.x.ai/developers/tools/web-search
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "xai"           # explicit per-capability
+      backend: "xai"                  # shared fallback
+
+Optional knobs (under ``web.xai`` in ``config.yaml``)::
+
+    web:
+      xai:
+        model: "grok-4.3"             # reasoning model required by web_search
+        allowed_domains: ["x.ai"]     # max 5 — mutually exclusive with excluded_domains
+        excluded_domains: ["bad.com"] # max 5 — mutually exclusive with allowed_domains
+        timeout: 90                   # seconds (default 90)
+
+Auth: reuses :func:`tools.xai_http.resolve_xai_http_credentials`, which
+prefers Hermes-managed xAI Grok OAuth (via ``hermes auth``) and falls back
+to ``XAI_API_KEY`` (resolved through ``~/.hermes/.env``, then
+``os.environ``).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional
+
+from agent.web_search_provider import WebSearchProvider
+from tools.xai_http import (
+    has_xai_credentials,
+    hermes_xai_user_agent,
+    resolve_xai_http_credentials,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MODEL = "grok-4.3"
+DEFAULT_TIMEOUT = 90
+_MAX_DOMAIN_FILTERS = 5  # xAI hard cap on allowed_domains / excluded_domains
+
+# Match the JSON object Grok is asked to emit. Tolerates leading/trailing
+# prose since reasoning models occasionally narrate before the JSON block
+# even when explicitly asked not to.
+_JSON_BLOCK_RE = re.compile(r"\{[\s\S]*\}", re.MULTILINE)
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+def _load_xai_web_config() -> Dict[str, Any]:
+    """Read ``web.xai`` from config.yaml (returns {} on miss)."""
+    try:
+        from hermes_cli.config import load_config
+
+        cfg = load_config()
+        web_section = cfg.get("web") if isinstance(cfg, dict) else None
+        xai_section = web_section.get("xai") if isinstance(web_section, dict) else None
+        return xai_section if isinstance(xai_section, dict) else {}
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Could not load web.xai config: %s", exc)
+        return {}
+
+
+def _coerce_domain_list(value: Any) -> List[str]:
+    """Coerce a config value to a clean list of <=5 domain strings."""
+    if not isinstance(value, list):
+        return []
+    cleaned: List[str] = []
+    for item in value:
+        if isinstance(item, str) and item.strip():
+            cleaned.append(item.strip())
+        if len(cleaned) >= _MAX_DOMAIN_FILTERS:
+            break
+    return cleaned
+
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+
+class XAIWebSearchProvider(WebSearchProvider):
+    """Search-only provider backed by xAI's agentic Web Search tool.
+
+    Sends a structured prompt to Grok with ``tools=[{"type": "web_search"}]``
+    enabled and asks it to return the top *limit* results as JSON. Falls
+    back to the Responses API ``citations`` list if Grok ignores the JSON
+    schema instruction (rare for grok-4.3 but cheap insurance).
+
+    No extract capability — pair with Firecrawl / Tavily / Exa for
+    ``web_extract`` if you need page content.
+
+    Trust model
+    -----------
+    Unlike index-backed providers (Brave / Tavily / Exa) which return
+    verbatim search-engine results, this backend is an LLM in a trench
+    coat: Grok decides which URLs to surface, generates the titles and
+    descriptions itself, and is influenced by the *content of the query*.
+    A maliciously crafted query (e.g. injected via untrusted upstream
+    input the agent picked up) can in principle steer Grok into emitting
+    attacker-chosen URLs. Callers that pipe untrusted text directly into
+    ``web_search`` should treat returned URLs the same way they would
+    treat any model-generated link — validate before fetching.
+    """
+
+    @property
+    def name(self) -> str:
+        return "xai"
+
+    @property
+    def display_name(self) -> str:
+        return "xAI Web Search (Grok)"
+
+    def is_available(self) -> bool:
+        """Cheap availability probe — env var OR auth-store has OAuth tokens.
+
+        Delegates to :func:`tools.xai_http.has_xai_credentials`, which is
+        deliberately *not* the same as :func:`resolve_xai_http_credentials`:
+        it never triggers OAuth token refresh or acquires the auth-store
+        lock. The ABC contract requires this method to be safe to call on
+        every ``hermes tools`` repaint and at tool-registration time.
+        Token freshness / refresh is handled inside :meth:`search`.
+        """
+        return has_xai_credentials()
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    # -- Search -----------------------------------------------------------
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Grok-backed web search.
+
+        Returns ``{"success": True, "data": {"web": [{title, url, description, position}, ...]}}``
+        on success, ``{"success": False, "error": str}`` on failure.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+        except Exception:  # noqa: BLE001 — interrupt module is best-effort
+            pass
+
+        creds = resolve_xai_http_credentials()
+        api_key = str(creds.get("api_key") or "").strip()
+        base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
+        if not api_key:
+            return {
+                "success": False,
+                "error": (
+                    "No xAI credentials found. Run `hermes auth` to sign in with "
+                    "xAI Grok OAuth, or set XAI_API_KEY."
+                ),
+            }
+
+        # Clamp limit to the same range the caller (web_search_tool) accepts,
+        # so we don't silently downgrade explicit limits. Grok happily
+        # produces longer lists; cost scales linearly with the requested
+        # count via reasoning tokens, but that's the caller's call to make.
+        try:
+            limit = int(limit)
+        except (TypeError, ValueError):
+            limit = 5
+        limit = max(1, min(limit, 100))
+
+        cfg = _load_xai_web_config()
+        model = cfg.get("model") if isinstance(cfg.get("model"), str) else DEFAULT_MODEL
+        model = model.strip() or DEFAULT_MODEL
+
+        try:
+            timeout = float(cfg.get("timeout", DEFAULT_TIMEOUT))
+        except (TypeError, ValueError):
+            timeout = DEFAULT_TIMEOUT
+
+        allowed = _coerce_domain_list(cfg.get("allowed_domains"))
+        excluded = _coerce_domain_list(cfg.get("excluded_domains"))
+        if allowed and excluded:
+            # xAI explicitly rejects this combo — surface a clear error
+            # rather than a 400 from the API.
+            return {
+                "success": False,
+                "error": (
+                    "web.xai.allowed_domains and web.xai.excluded_domains "
+                    "cannot both be set (xAI restriction)."
+                ),
+            }
+
+        web_search_tool: Dict[str, Any] = {"type": "web_search"}
+        if allowed:
+            web_search_tool["filters"] = {"allowed_domains": allowed}
+        elif excluded:
+            web_search_tool["filters"] = {"excluded_domains": excluded}
+
+        prompt = self._build_prompt(query, limit)
+
+        payload: Dict[str, Any] = {
+            "model": model,
+            "input": [{"role": "user", "content": prompt}],
+            "tools": [web_search_tool],
+            # Drop inline citation markdown — we want the JSON block clean,
+            # and we read URLs from annotations / citations separately.
+            "include": ["no_inline_citations"],
+        }
+
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": hermes_xai_user_agent(),
+        }
+
+        try:
+            import httpx
+        except ImportError:
+            return {
+                "success": False,
+                "error": "httpx is not installed (required for xAI web search)",
+            }
+
+        logger.info(
+            "xAI web search via %s: '%s' (limit=%d, model=%s)",
+            base_url, query, limit, model,
+        )
+
+        # Two-attempt loop: if the first call returns 401 and our creds came
+        # from the OAuth path, force-refresh the token once and retry. This
+        # closes two gaps the proactive resolver check doesn't cover:
+        # (1) opaque (non-JWT) access tokens — `_xai_access_token_is_expiring`
+        #     can't decode them and returns False, so refresh never fires
+        #     until the server hands us a 401.
+        # (2) mid-window revocation — admin revoke, refresh-token rotation,
+        #     or clock skew can produce 401s on a token whose JWT `exp` claim
+        #     is still in the future.
+        # Env-var (`XAI_API_KEY`) credentials skip the retry entirely — we
+        # can't refresh those and an immediate retry would just burn quota.
+        is_oauth_path = (creds.get("provider") == "xai-oauth")
+        resp = None
+        for attempt in range(2):
+            try:
+                resp = httpx.post(
+                    f"{base_url}/responses",
+                    headers=headers,
+                    json=payload,
+                    timeout=timeout,
+                )
+                resp.raise_for_status()
+                break
+            except httpx.HTTPStatusError as exc:
+                status = exc.response.status_code if exc.response is not None else 0
+                if status == 401 and attempt == 0 and is_oauth_path:
+                    logger.info(
+                        "xAI web search got 401 on first attempt; forcing OAuth "
+                        "refresh and retrying once.",
+                    )
+                    try:
+                        refreshed = resolve_xai_http_credentials(force_refresh=True)
+                        refreshed_key = str(refreshed.get("api_key") or "").strip()
+                        if refreshed_key and refreshed_key != api_key:
+                            api_key = refreshed_key
+                            headers["Authorization"] = f"Bearer {api_key}"
+                            continue
+                        # Refresh returned the same (or empty) token — no point
+                        # in retrying. Fall through to the error return below.
+                    except Exception as refresh_exc:  # noqa: BLE001
+                        logger.warning(
+                            "xAI web search OAuth refresh after 401 failed: %s",
+                            refresh_exc,
+                        )
+                body = ""
+                try:
+                    body = exc.response.text[:300] if exc.response is not None else ""
+                except Exception:
+                    body = ""
+                logger.warning("xAI web search HTTP %d: %s", status, body)
+                return {
+                    "success": False,
+                    "error": f"xAI web search returned HTTP {status}: {body}".rstrip(),
+                }
+            except httpx.RequestError as exc:
+                logger.warning("xAI web search request error: %s", exc)
+                return {"success": False, "error": f"Could not reach xAI: {exc}"}
+
+        if resp is None:
+            # Defensive — both attempts somehow exited the loop without resp.
+            return {"success": False, "error": "xAI web search produced no response"}
+
+        try:
+            data = resp.json()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("xAI web search bad JSON: %s", exc)
+            return {
+                "success": False,
+                "error": "Could not parse xAI Responses API reply as JSON",
+            }
+
+        # xAI's Responses surface sometimes returns HTTP 200 with an error
+        # envelope (model overloaded, content-policy refusal, etc.). Without
+        # this check, ``_extract_results`` would silently produce an empty
+        # list and we'd report success-with-no-rows — masking a real failure
+        # the agent should see and decide whether to retry.
+        api_error = data.get("error") if isinstance(data, dict) else None
+        if isinstance(api_error, dict):
+            err_msg = (
+                api_error.get("message")
+                or api_error.get("code")
+                or "unknown error"
+            )
+            logger.warning("xAI web search returned error envelope: %s", err_msg)
+            return {"success": False, "error": f"xAI returned an error: {err_msg}"}
+
+        web_results = self._extract_results(data, limit=limit)
+        if not web_results:
+            # Successful call, just no usable rows — return success with an
+            # empty list so the model can decide whether to retry. Matches
+            # what brave-free / exa do when the upstream API returns 0 hits.
+            return {"success": True, "data": {"web": []}}
+
+        return {"success": True, "data": {"web": web_results}}
+
+    # -- Prompt + parsing -------------------------------------------------
+
+    @staticmethod
+    def _build_prompt(query: str, limit: int) -> str:
+        """Compose the prompt that asks Grok to act as a search engine.
+
+        We deliberately ask for a JSON object (not bare array) so we can
+        match it cheaply with ``_JSON_BLOCK_RE``; we explicitly forbid
+        prose, markdown fences, and inline-citation links to keep the
+        payload parseable.
+        """
+        return (
+            "Use the web_search tool to find current information for the query below, "
+            "then respond with ONLY a single JSON object — no prose, no markdown "
+            "fences, no inline citation links — matching this exact schema:\n\n"
+            '{"results": [{"title": "string", "url": "string", '
+            '"description": "1-2 sentence summary"}]}\n\n'
+            f'Return at most {limit} results, ordered by relevance, with absolute '
+            "https:// URLs. If no usable results exist, return "
+            '{"results": []}.\n\n'
+            f"Query: {query}"
+        )
+
+    @classmethod
+    def _extract_results(
+        cls,
+        response_data: Dict[str, Any],
+        *,
+        limit: int,
+    ) -> List[Dict[str, Any]]:
+        """Pull a ``[{title, url, description, position}, ...]`` list out of a
+        Responses-API reply.
+
+        Strategy:
+
+        1. Walk ``output[*].content[*].text`` for ``output_text`` blocks and
+           try to parse the first JSON object that has a ``results`` list.
+        2. If the JSON path fails, fall back to the message annotations
+           (``url_citation`` entries) — every annotation carries a URL and
+           a ``title`` (citation number); we pair those URLs with surrounding
+           text from the message body as a best-effort description.
+        """
+        text_blocks, annotations = cls._collect_output_text(response_data)
+
+        # Primary path: parse the JSON object Grok was asked for.
+        for block in text_blocks:
+            parsed = cls._try_parse_json_results(block, limit=limit)
+            if parsed:
+                return parsed
+
+        # Secondary path: derive results from message annotations + raw text.
+        # Only short-circuit when annotations actually yielded usable rows;
+        # otherwise fall through to the citations list. (xAI currently only
+        # emits ``url_citation`` annotations, but future annotation types
+        # would silently produce an empty result set if we returned here
+        # unconditionally — masking real data in ``citations``.)
+        if annotations:
+            joined_text = "\n".join(text_blocks)
+            annotation_results = cls._results_from_annotations(
+                annotations, joined_text, limit=limit,
+            )
+            if annotation_results:
+                return annotation_results
+
+        # Last-ditch: raw citations list (no titles or descriptions).
+        citations = response_data.get("citations") or []
+        if isinstance(citations, list):
+            return [
+                {
+                    "title": "",
+                    "url": str(u),
+                    "description": "",
+                    "position": i + 1,
+                }
+                for i, u in enumerate(citations[:limit])
+                if isinstance(u, str) and u.strip()
+            ]
+
+        return []
+
+    @staticmethod
+    def _collect_output_text(
+        response_data: Dict[str, Any],
+    ) -> tuple[List[str], List[Dict[str, Any]]]:
+        """Return (text_blocks, annotations) extracted from ``response.output``."""
+        text_blocks: List[str] = []
+        annotations: List[Dict[str, Any]] = []
+        output = response_data.get("output")
+        if not isinstance(output, list):
+            return text_blocks, annotations
+
+        for item in output:
+            if not isinstance(item, dict) or item.get("type") != "message":
+                continue
+            content = item.get("content")
+            if not isinstance(content, list):
+                continue
+            for chunk in content:
+                if not isinstance(chunk, dict) or chunk.get("type") != "output_text":
+                    continue
+                text = chunk.get("text")
+                if isinstance(text, str) and text.strip():
+                    text_blocks.append(text)
+                chunk_annotations = chunk.get("annotations")
+                if isinstance(chunk_annotations, list):
+                    for ann in chunk_annotations:
+                        if isinstance(ann, dict):
+                            annotations.append(ann)
+        return text_blocks, annotations
+
+    @staticmethod
+    def _try_parse_json_results(
+        text: str,
+        *,
+        limit: int,
+    ) -> Optional[List[Dict[str, Any]]]:
+        """Parse a JSON object with a ``results`` array out of ``text``.
+
+        Returns the normalized result list on success, ``None`` when the
+        block has no valid JSON object or no ``results`` key. Tolerates
+        leading/trailing prose because reasoning models sometimes prefix a
+        short narration even when told not to.
+        """
+        # Try the whole string first — cheapest path when Grok obeys.
+        candidates = [text]
+        match = _JSON_BLOCK_RE.search(text)
+        if match and match.group(0) != text:
+            candidates.append(match.group(0))
+
+        for candidate in candidates:
+            try:
+                parsed = json.loads(candidate)
+            except (json.JSONDecodeError, ValueError):
+                continue
+            if not isinstance(parsed, dict):
+                continue
+            results = parsed.get("results")
+            if not isinstance(results, list):
+                continue
+            normalized: List[Dict[str, Any]] = []
+            for row in results[:limit]:
+                if not isinstance(row, dict):
+                    continue
+                url = str(row.get("url", "")).strip()
+                if not url:
+                    continue
+                normalized.append(
+                    {
+                        "title": str(row.get("title", "")).strip(),
+                        "url": url,
+                        "description": str(row.get("description", "")).strip(),
+                        # Renumber from the kept results, not the raw input
+                        # index, so a dropped malformed row doesn't leave a
+                        # gap in the positions handed back to the agent.
+                        "position": len(normalized) + 1,
+                    }
+                )
+            if normalized:
+                return normalized
+        return None
+
+    @staticmethod
+    def _results_from_annotations(
+        annotations: List[Dict[str, Any]],
+        joined_text: str,
+        *,
+        limit: int,
+    ) -> List[Dict[str, Any]]:
+        """Best-effort fallback when JSON parsing fails.
+
+        Uses each ``url_citation`` annotation's ``url`` (the citation
+        title is just the integer label, so we don't surface it) and
+        slices ~200 characters of surrounding text as the description.
+        """
+        seen: set[str] = set()
+        results: List[Dict[str, Any]] = []
+        for ann in annotations:
+            if ann.get("type") != "url_citation":
+                continue
+            url = str(ann.get("url", "")).strip()
+            if not url or url in seen:
+                continue
+            seen.add(url)
+
+            description = ""
+            start = ann.get("start_index")
+            end = ann.get("end_index")
+            if isinstance(start, int) and isinstance(end, int) and 0 <= start < end <= len(joined_text):
+                window_start = max(0, start - 200)
+                description = joined_text[window_start:start].strip()
+                if len(description) > 200:
+                    description = description[-200:].strip()
+
+            results.append(
+                {
+                    "title": "",
+                    "url": url,
+                    "description": description,
+                    "position": len(results) + 1,
+                }
+            )
+            if len(results) >= limit:
+                break
+        return results
+
+    # -- Setup picker -----------------------------------------------------
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        # Auth resolution is delegated to the shared ``xai_grok`` post_setup
+        # hook (same one image_gen.xai and tts.xai use) so users see the
+        # familiar OAuth-or-API-key prompt for every xAI service.
+        return {
+            "name": "xAI Web Search (Grok)",
+            "badge": "paid",
+            "tag": (
+                "Agentic web search via Grok's web_search tool — uses xAI "
+                "Grok OAuth or XAI_API_KEY."
+            ),
+            "env_vars": [],
+            "post_setup": "xai_grok",
+        }