Sami/hermes-agent
1
0
Fork 0
forked from Zakaria/hermes-agent
Code Pull Requests Activity

Hermes-agent

Browse Source
This commit is contained in:
Zakaria
2026-06-14 14:30:48 -04:00
commit dac4b88b94
5058 changed files with 1884848 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/computer_use/__init__.py
+43
View File
@@ -0,0 +1,43 @@
"""Computer use toolset — universal (any-model) macOS desktop control.
Architecture
------------
This toolset drives macOS apps through cua-driver's background computer-use
primitive (SkyLight private SPIs for focus-without-raise + pid-scoped event
posting). Unlike #4562's pyautogui backend, it does NOT steal the user's
cursor, keyboard focus, or Space — the agent and the user can co-work on the
same machine.
Unlike #4562's Anthropic-native `computer_20251124` tool, the schema here is
a plain OpenAI function-calling schema that every tool-capable model can
drive. Vision models get SOM (set-of-mark) captures — a screenshot with
numbered overlays on every interactable element plus the AX tree — so they
click by element index instead of pixel coordinates. Non-vision models can
drive via the AX tree alone.
Wiring
------
* `tool.py` — registers the `computer_use` tool via tools.registry.
* `backend.py` — abstract `ComputerUseBackend`; swappable implementation.
* `cua_backend.py`— default backend; speaks MCP over stdio to `cua-driver`.
* `schema.py` — shared schema + docstring for the generic `computer_use`
tool. Model-agnostic.
* `capture.py` — screenshot post-processing (PNG coercion, sizing, SOM
overlay if the backend did not).
The outer integration points (multimodal tool-result plumbing, screenshot
eviction in the Anthropic adapter, image-aware token estimation, the
COMPUTER_USE_GUIDANCE prompt block, approval hook, and the skill) live
alongside this package. See agent/anthropic_adapter.py and
agent/prompt_builder.py for the salvaged hunks from PR #4562.
"""
from __future__ import annotations
# Re-export the public surface so `from tools.computer_use import ...` works.
from tools.computer_use.tool import ( # noqa: F401
handle_computer_use,
set_approval_callback,
check_computer_use_requirements,
get_computer_use_schema,
)
tools/computer_use/backend.py
+158
View File
@@ -0,0 +1,158 @@
"""Abstract backend interface for computer use.
Any implementation (cua-driver over MCP, pyautogui, noop, future Linux/Windows)
must return the shape described below. All methods synchronous; async is
handled inside the backend implementation if needed.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
@dataclass
class UIElement:
"""One interactable element on the current screen."""
index: int # 1-based SOM index
role: str # AX role (AXButton, AXTextField, ...)
label: str = "" # AXTitle / AXDescription / AXValue snippet
bounds: Tuple[int, int, int, int] = (0, 0, 0, 0) # x, y, w, h (logical px)
app: str = "" # owning bundle ID or app name
pid: int = 0 # owning process PID
window_id: int = 0 # SkyLight / CG window ID
attributes: Dict[str, Any] = field(default_factory=dict)
def center(self) -> Tuple[int, int]:
x, y, w, h = self.bounds
return x + w // 2, y + h // 2
@dataclass
class CaptureResult:
"""Result of a screen capture call.
At least one of png_b64 / elements is populated depending on capture mode:
* mode="vision" → png_b64 only
* mode="ax" → elements only
* mode="som" → both (default): PNG already has numbered overlays
drawn by the backend, and `elements` holds the
matching index → element mapping.
"""
mode: str
width: int # screenshot width (logical px, pre-Anthropic-scale)
height: int
png_b64: Optional[str] = None
elements: List[UIElement] = field(default_factory=list)
# Optional: the target app/window the elements were captured for.
app: str = ""
window_title: str = ""
# Raw bytes we sent to Anthropic, for token estimation.
png_bytes_len: int = 0
@dataclass
class ActionResult:
"""Result of any action (click / type / scroll / drag / key / wait)."""
ok: bool
action: str
message: str = "" # human-readable summary
# Optional trailing screenshot — set when the caller asked for a
# post-action capture or the backend always returns one.
capture: Optional[CaptureResult] = None
# Arbitrary extra fields for debugging / telemetry.
meta: Dict[str, Any] = field(default_factory=dict)
class ComputerUseBackend(ABC):
"""Lifecycle: `start()` before first use, `stop()` at shutdown."""
@abstractmethod
def start(self) -> None: ...
@abstractmethod
def stop(self) -> None: ...
@abstractmethod
def is_available(self) -> bool:
"""Return True if the backend can be used on this host right now.
Used by check_fn gating and by the post-setup wizard.
"""
# ── Capture ─────────────────────────────────────────────────────
@abstractmethod
def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult: ...
# ── Pointer actions ─────────────────────────────────────────────
@abstractmethod
def click(
self,
*,
element: Optional[int] = None,
x: Optional[int] = None,
y: Optional[int] = None,
button: str = "left", # left | right | middle
click_count: int = 1,
modifiers: Optional[List[str]] = None,
) -> ActionResult: ...
@abstractmethod
def drag(
self,
*,
from_element: Optional[int] = None,
to_element: Optional[int] = None,
from_xy: Optional[Tuple[int, int]] = None,
to_xy: Optional[Tuple[int, int]] = None,
button: str = "left",
modifiers: Optional[List[str]] = None,
) -> ActionResult: ...
@abstractmethod
def scroll(
self,
*,
direction: str, # up | down | left | right
amount: int = 3, # wheel ticks
element: Optional[int] = None,
x: Optional[int] = None,
y: Optional[int] = None,
modifiers: Optional[List[str]] = None,
) -> ActionResult: ...
# ── Keyboard ────────────────────────────────────────────────────
@abstractmethod
def type_text(self, text: str) -> ActionResult: ...
@abstractmethod
def key(self, keys: str) -> ActionResult:
"""Send a key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return'."""
# ── Introspection ───────────────────────────────────────────────
@abstractmethod
def list_apps(self) -> List[Dict[str, Any]]:
"""Return running apps with bundle IDs, PIDs, window counts."""
@abstractmethod
def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
"""Route input to `app` (by name or bundle ID). Default: focus without raise."""
# ── Native-value mutation ────────────────────────────────────────
@abstractmethod
def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
"""Set a native value on an element (e.g. AXPopUpButton selection).
`element` is the 1-based SOM index returned by a prior capture call.
"""
# ── Timing ──────────────────────────────────────────────────────
def wait(self, seconds: float) -> ActionResult:
"""Default implementation: time.sleep."""
import time
time.sleep(max(0.0, min(seconds, 30.0)))
return ActionResult(ok=True, action="wait", message=f"waited {seconds:.2f}s")
tools/computer_use/cua_backend.py
+779
View File
@@ -0,0 +1,779 @@
"""Cua-driver backend (macOS only).
Speaks MCP over stdio to `cua-driver`. The Python `mcp` SDK is async, so we
run a dedicated asyncio event loop on a background thread and marshal sync
calls through it.
Install: `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)"`
After install, `cua-driver` is on $PATH and supports `cua-driver mcp` (stdio
transport) which is what we invoke.
The private SkyLight SPIs cua-driver uses (SLEventPostToPid, SLPSPostEvent-
RecordTo, _AXObserverAddNotificationAndCheckRemote) are not Apple-public and
can break on OS updates. Pin the installed version via `HERMES_CUA_DRIVER_
VERSION` if you want reproducibility across an OS bump.
"""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import os
import re
import shutil
import sys
import threading
from typing import Any, Dict, List, Optional, Tuple
from tools.computer_use.backend import (
ActionResult,
CaptureResult,
ComputerUseBackend,
UIElement,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Version pinning
# ---------------------------------------------------------------------------
PINNED_CUA_DRIVER_VERSION = os.environ.get("HERMES_CUA_DRIVER_VERSION", "0.5.0")
_CUA_DRIVER_CMD = os.environ.get("HERMES_CUA_DRIVER_CMD", "cua-driver")
_CUA_DRIVER_ARGS = ["mcp"] # stdio MCP transport
# Regex to parse list_windows text output lines:
# "- AppName (pid 12345) "Title" [window_id: 67890]"
_WINDOW_LINE_RE = re.compile(
r'^-\s+(.+?)\s+\(pid\s+(\d+)\)\s+.*\[window_id:\s+(\d+)\]',
re.MULTILINE,
)
# Regex to parse element lines from get_window_state AX tree markdown.
#
# Handles two output formats from different cua-driver versions:
# Classic: " - [N] AXRole \"label\""
# New: "[N] AXRole (order) id=Label"
#
# Group 1: element index
# Group 2: AX role
# Group 3: quoted label (classic format)
# Group 4: id= label (new format)
_ELEMENT_LINE_RE = re.compile(
r'^\s*(?:-\s+)?\[(\d+)\]\s+(\w+)(?:\s+"([^"]*)"|(?:\s+\(\d+\))?\s+id=([^\s\[\]]*))?' ,
re.MULTILINE,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _is_macos() -> bool:
return sys.platform == "darwin"
def cua_driver_binary_available() -> bool:
"""True if `cua-driver` is on $PATH or HERMES_CUA_DRIVER_CMD resolves."""
return bool(shutil.which(_CUA_DRIVER_CMD))
def cua_driver_install_hint() -> str:
return (
"cua-driver is not installed. Install with one of:\n"
" hermes computer-use install\n"
"Or run the upstream installer directly:\n"
' /bin/bash -c "$(curl -fsSL '
'https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)"\n'
"Or run `hermes tools` and enable the Computer Use toolset to install it automatically."
)
def _parse_windows_from_text(text: str) -> List[Dict[str, Any]]:
"""Parse window records from list_windows text output."""
windows = []
for m in _WINDOW_LINE_RE.finditer(text):
windows.append({
"app_name": m.group(1).strip(),
"pid": int(m.group(2)),
"window_id": int(m.group(3)),
"off_screen": "[off-screen]" in m.group(0),
})
return windows
def _parse_elements_from_tree(markdown: str) -> List[UIElement]:
"""Parse UIElement list from get_window_state AX tree markdown.
Handles both the classic ``"label"``-quoted format and the newer
``id=Label`` format introduced in cua-driver v0.1.6.
"""
elements = []
for m in _ELEMENT_LINE_RE.finditer(markdown):
# group(3) = quoted label (classic); group(4) = id= label (new)
label = m.group(3) or m.group(4) or ""
elements.append(UIElement(
index=int(m.group(1)),
role=m.group(2),
label=label,
bounds=(0, 0, 0, 0),
))
return elements
def _image_dimensions_from_bytes(raw: bytes) -> Tuple[int, int]:
"""Best-effort PNG/JPEG dimension sniffing without extra dependencies."""
if raw.startswith(b"\x89PNG\r\n\x1a\n") and len(raw) >= 24:
width = int.from_bytes(raw[16:20], "big")
height = int.from_bytes(raw[20:24], "big")
if width > 0 and height > 0:
return width, height
if raw.startswith(b"\xff\xd8"):
i = 2
n = len(raw)
while i + 9 < n:
if raw[i] != 0xFF:
i += 1
continue
marker = raw[i + 1]
i += 2
if marker in {0xD8, 0xD9} or 0xD0 <= marker <= 0xD7:
continue
if i + 2 > n:
break
segment_len = int.from_bytes(raw[i:i + 2], "big")
if segment_len < 2 or i + segment_len > n:
break
if marker in {
0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7,
0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF,
}:
if segment_len >= 7:
height = int.from_bytes(raw[i + 3:i + 5], "big")
width = int.from_bytes(raw[i + 5:i + 7], "big")
if width > 0 and height > 0:
return width, height
break
i += segment_len
return 0, 0
def _split_tree_text(full_text: str) -> Tuple[str, str]:
"""Split get_window_state text into (summary_line, tree_markdown)."""
lines = full_text.split("\n", 1)
summary = lines[0]
tree = lines[1] if len(lines) > 1 else ""
return summary, tree
def _parse_key_combo(keys: str) -> Tuple[Optional[str], List[str]]:
"""Parse a key string like 'cmd+s' into (key, modifiers).
Returns (key, modifiers) where key is the non-modifier key and modifiers
is a list of modifier names (cmd, shift, option, ctrl).
"""
MODIFIER_NAMES = {"cmd", "command", "shift", "option", "alt", "ctrl", "control", "fn"}
KEY_ALIASES = {"command": "cmd", "alt": "option", "control": "ctrl"}
parts = [p.strip().lower() for p in re.split(r'[+\-]', keys) if p.strip()]
modifiers = []
key = None
for part in parts:
normalized = KEY_ALIASES.get(part, part)
if normalized in MODIFIER_NAMES:
modifiers.append(normalized)
else:
key = part # last non-modifier wins
return key, modifiers
# ---------------------------------------------------------------------------
# Asyncio bridge — one long-lived loop on a background thread
# ---------------------------------------------------------------------------
class _AsyncBridge:
"""Runs one asyncio loop on a daemon thread; marshals coroutines from the caller."""
def __init__(self) -> None:
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._thread: Optional[threading.Thread] = None
self._ready = threading.Event()
def start(self) -> None:
if self._thread and self._thread.is_alive():
return
self._ready.clear()
def _run() -> None:
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._ready.set()
try:
self._loop.run_forever()
finally:
try:
self._loop.close()
except Exception:
pass
self._thread = threading.Thread(target=_run, daemon=True, name="cua-driver-loop")
self._thread.start()
if not self._ready.wait(timeout=5.0):
raise RuntimeError("cua-driver asyncio bridge failed to start")
def run(self, coro, timeout: Optional[float] = 30.0) -> Any:
from agent.async_utils import safe_schedule_threadsafe
if not self._loop or not self._thread or not self._thread.is_alive():
if asyncio.iscoroutine(coro):
coro.close()
raise RuntimeError("cua-driver bridge not started")
fut = safe_schedule_threadsafe(coro, self._loop)
if fut is None:
raise RuntimeError("cua-driver bridge not started")
return fut.result(timeout=timeout)
def stop(self) -> None:
if self._loop and self._loop.is_running():
self._loop.call_soon_threadsafe(self._loop.stop)
if self._thread:
self._thread.join(timeout=2.0)
self._thread = None
self._loop = None
# ---------------------------------------------------------------------------
# MCP session (lazy, shared across tool calls)
# ---------------------------------------------------------------------------
class _CuaDriverSession:
"""Holds the mcp ClientSession. Spawned lazily; re-entered on drop."""
def __init__(self, bridge: _AsyncBridge) -> None:
self._bridge = bridge
self._session = None
self._exit_stack = None
self._lock = threading.Lock()
self._started = False
def _require_started(self) -> None:
if not self._started:
raise RuntimeError("cua-driver session not started")
async def _aenter(self) -> None:
from contextlib import AsyncExitStack
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
if not cua_driver_binary_available():
raise RuntimeError(cua_driver_install_hint())
params = StdioServerParameters(
command=_CUA_DRIVER_CMD,
args=_CUA_DRIVER_ARGS,
env={**os.environ},
)
stack = AsyncExitStack()
read, write = await stack.enter_async_context(stdio_client(params))
session = await stack.enter_async_context(ClientSession(read, write))
await session.initialize()
self._exit_stack = stack
self._session = session
async def _aexit(self) -> None:
if self._exit_stack is not None:
try:
await self._exit_stack.aclose()
except Exception as e:
logger.warning("cua-driver shutdown error: %s", e)
self._exit_stack = None
self._session = None
def start(self) -> None:
with self._lock:
if self._started:
return
self._bridge.start()
self._bridge.run(self._aenter(), timeout=15.0)
self._started = True
def stop(self) -> None:
with self._lock:
if not self._started:
return
try:
self._bridge.run(self._aexit(), timeout=5.0)
finally:
self._started = False
async def _call_tool_async(self, name: str, args: Dict[str, Any]) -> Dict[str, Any]:
result = await self._session.call_tool(name, args)
return _extract_tool_result(result)
@staticmethod
def _is_closed_session_error(exc: Exception) -> bool:
"""Return True for MCP/stdio failures that are recoverable by reconnecting."""
name = exc.__class__.__name__
module = getattr(exc.__class__, "__module__", "")
return (
name in {"ClosedResourceError", "BrokenResourceError", "EndOfStream"}
or (module.startswith("anyio") and "Resource" in name)
or isinstance(exc, (BrokenPipeError, EOFError))
)
def _restart_session_locked(self) -> None:
"""Recreate the MCP session after the daemon/stdin transport was closed."""
try:
if self._started:
self._bridge.run(self._aexit(), timeout=5.0)
except Exception as e:
logger.debug("cua-driver session cleanup before reconnect failed: %s", e)
self._started = False
self._bridge.run(self._aenter(), timeout=15.0)
self._started = True
def call_tool(self, name: str, args: Dict[str, Any], timeout: float = 30.0) -> Dict[str, Any]:
self._require_started()
try:
return self._bridge.run(self._call_tool_async(name, args), timeout=timeout)
except Exception as e:
if not self._is_closed_session_error(e):
raise
# Daemon restart closes the cached stdio channel. Reconnect once and
# retry exactly one more time — never loop, to avoid hammering a
# genuinely dead daemon.
logger.warning("cua-driver MCP session closed during %s; reconnecting once", name)
with self._lock:
self._restart_session_locked()
return self._bridge.run(self._call_tool_async(name, args), timeout=timeout)
def _extract_tool_result(mcp_result: Any) -> Dict[str, Any]:
"""Convert an mcp CallToolResult into a plain dict.
cua-driver returns a mix of text parts, image parts, and structuredContent.
We flatten into:
{
"data": <text or parsed json>,
"images": [b64, ...],
"structuredContent": <dict|None>,
"isError": bool,
}
structuredContent is populated from the MCP result's structuredContent field
(MCP spec §2024-11-05+) and takes precedence for structured data like
list_windows window arrays.
"""
data: Any = None
images: List[str] = []
is_error = bool(getattr(mcp_result, "isError", False))
structured: Optional[Dict] = getattr(mcp_result, "structuredContent", None) or None
text_chunks: List[str] = []
for part in getattr(mcp_result, "content", []) or []:
ptype = getattr(part, "type", None)
if ptype == "text":
text_chunks.append(getattr(part, "text", "") or "")
elif ptype == "image":
b64 = getattr(part, "data", None)
if b64:
images.append(b64)
if text_chunks:
joined = "\n".join(t for t in text_chunks if t)
try:
data = json.loads(joined) if joined.strip().startswith(("{", "[")) else joined
except json.JSONDecodeError:
data = joined
return {"data": data, "images": images, "structuredContent": structured, "isError": is_error}
# ---------------------------------------------------------------------------
# The backend itself
# ---------------------------------------------------------------------------
class CuaDriverBackend(ComputerUseBackend):
"""Default computer-use backend. macOS-only via cua-driver MCP."""
def __init__(self) -> None:
self._bridge = _AsyncBridge()
self._session = _CuaDriverSession(self._bridge)
# Sticky context — updated by capture(), used by action tools.
self._active_pid: Optional[int] = None
self._active_window_id: Optional[int] = None
self._last_app: Optional[str] = None # last app name targeted via capture/focus_app
# ── Lifecycle ──────────────────────────────────────────────────
def start(self) -> None:
self._session.start()
def stop(self) -> None:
try:
self._session.stop()
finally:
self._bridge.stop()
def is_available(self) -> bool:
if not _is_macos():
return False
return cua_driver_binary_available()
# ── Capture ────────────────────────────────────────────────────
def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult:
"""Capture the frontmost on-screen window (optionally filtered by app name).
Maps hermes `capture(mode, app)` → cua-driver `list_windows` +
`get_window_state` (ax/som) or `screenshot` (vision).
"""
# Step 1: enumerate on-screen windows to find target pid/window_id.
lw_out = self._session.call_tool("list_windows", {"on_screen_only": True})
# Prefer structuredContent.windows (MCP 2024-11-05+); fall back to
# text-line parsing for older cua-driver builds.
sc = lw_out.get("structuredContent") or {}
raw_windows = sc.get("windows") if sc else None
if raw_windows:
windows = [
{
"app_name": w.get("app_name", ""),
"pid": int(w["pid"]),
"window_id": int(w["window_id"]),
"off_screen": not w.get("is_on_screen", True),
"title": w.get("title", ""),
"z_index": w.get("z_index", 0),
}
for w in raw_windows
]
# Sort by z_index descending (lowest z_index = frontmost on macOS).
windows.sort(key=lambda w: w["z_index"])
else:
raw_text = lw_out["data"] if isinstance(lw_out["data"], str) else ""
windows = _parse_windows_from_text(raw_text)
if not windows:
return CaptureResult(mode=mode, width=0, height=0, png_b64=None,
elements=[], app="", window_title="", png_bytes_len=0)
# Filter by app name (case-insensitive substring) if requested.
# When the filter matches nothing, surface that explicitly instead of
# silently capturing the frontmost window — on macOS the `app_name`
# returned by list_windows is the localized name (e.g. "計算機"), so
# `app="Calculator"` legitimately matches no windows on a non-English
# system and the caller needs to retry with the localized name.
if app:
app_lower = app.lower()
filtered = [w for w in windows if app_lower in w["app_name"].lower()]
if not filtered:
return CaptureResult(
mode=mode, width=0, height=0, png_b64=None,
elements=[], app="",
window_title=(
f"<no on-screen window matched app={app!r}; "
f"call list_apps to see available app names "
f"(macOS reports localized names, e.g. '計算機' "
f"instead of 'Calculator')>"
),
png_bytes_len=0,
)
windows = filtered
# Pick first on-screen window (sorted by z_index / z-order above).
target = next((w for w in windows if not w["off_screen"]), windows[0])
self._active_pid = target["pid"]
self._active_window_id = target["window_id"]
app_name = target["app_name"]
# Record the resolved app name so capture_after= follow-ups can re-target
# the same app rather than falling back to the frontmost window.
if app or not self._last_app:
self._last_app = app_name
# Step 2: capture.
png_b64: Optional[str] = None
elements: List[UIElement] = []
width = height = 0
window_title = ""
if mode == "vision":
# screenshot tool: just the PNG, no AX walk.
sc_out = self._session.call_tool(
"screenshot",
{"window_id": self._active_window_id, "format": "jpeg", "quality": 85},
)
if sc_out["images"]:
png_b64 = sc_out["images"][0]
else:
# get_window_state: AX tree + optional screenshot.
gws_out = self._session.call_tool(
"get_window_state",
{"pid": self._active_pid, "window_id": self._active_window_id},
)
text = gws_out["data"] if isinstance(gws_out["data"], str) else ""
summary, tree = _split_tree_text(text)
# Parse element count from summary e.g. "✅ AppName — 42 elements, turn 3..."
m = re.search(r'(\d+)\s+elements?', summary)
if tree and not gws_out["images"]:
# ax mode — no screenshot
elements = _parse_elements_from_tree(tree)
elif gws_out["images"]:
png_b64 = gws_out["images"][0]
elements = _parse_elements_from_tree(tree)
# Extract window title from the AX tree first AXWindow line.
wt = re.search(r'AXWindow\s+"([^"]+)"', tree)
if wt:
window_title = wt.group(1)
png_bytes_len = 0
if png_b64:
try:
raw = base64.b64decode(png_b64, validate=False)
png_bytes_len = len(raw)
detected_width, detected_height = _image_dimensions_from_bytes(raw)
if detected_width and detected_height:
width = detected_width
height = detected_height
except Exception:
png_bytes_len = len(png_b64) * 3 // 4
return CaptureResult(
mode=mode,
width=width,
height=height,
png_b64=png_b64,
elements=elements,
app=app_name,
window_title=window_title,
png_bytes_len=png_bytes_len,
)
# ── Pointer ────────────────────────────────────────────────────
def click(
self,
*,
element: Optional[int] = None,
x: Optional[int] = None,
y: Optional[int] = None,
button: str = "left",
click_count: int = 1,
modifiers: Optional[List[str]] = None,
) -> ActionResult:
pid = self._active_pid
if pid is None:
return ActionResult(ok=False, action="click",
message="No active window — call capture() first.")
# Choose tool based on button and click_count.
if button == "right":
tool = "right_click"
elif click_count == 2:
tool = "double_click"
else:
tool = "click"
args: Dict[str, Any] = {"pid": pid}
if element is not None:
if self._active_window_id is None:
return ActionResult(ok=False, action=tool,
message="No active window_id for element_index click.")
args["element_index"] = element
args["window_id"] = self._active_window_id
elif x is not None and y is not None:
args["x"] = x
args["y"] = y
else:
return ActionResult(ok=False, action=tool,
message="click requires element= or x/y.")
if modifiers:
args["modifier"] = modifiers
return self._action(tool, args)
def drag(
self,
*,
from_element: Optional[int] = None,
to_element: Optional[int] = None,
from_xy: Optional[Tuple[int, int]] = None,
to_xy: Optional[Tuple[int, int]] = None,
button: str = "left",
modifiers: Optional[List[str]] = None,
) -> ActionResult:
pid = self._active_pid
if pid is None:
return ActionResult(ok=False, action="drag",
message="No active window — call capture() first.")
args: Dict[str, Any] = {"pid": pid}
if from_element is not None and to_element is not None:
if self._active_window_id is None:
return ActionResult(ok=False, action="drag",
message="No active window_id for element-based drag.")
args["from_element"] = from_element
args["to_element"] = to_element
args["window_id"] = self._active_window_id
elif from_xy is not None and to_xy is not None:
args["from_x"], args["from_y"] = int(from_xy[0]), int(from_xy[1])
args["to_x"], args["to_y"] = int(to_xy[0]), int(to_xy[1])
else:
return ActionResult(ok=False, action="drag",
message="drag requires from_element/to_element or from_coordinate/to_coordinate.")
return self._action("drag", args)
def scroll(
self,
*,
direction: str,
amount: int = 3,
element: Optional[int] = None,
x: Optional[int] = None,
y: Optional[int] = None,
modifiers: Optional[List[str]] = None,
) -> ActionResult:
pid = self._active_pid
if pid is None:
return ActionResult(ok=False, action="scroll",
message="No active window — call capture() first.")
args: Dict[str, Any] = {
"pid": pid,
"direction": direction,
"amount": max(1, min(50, amount)),
}
if element is not None and self._active_window_id is not None:
args["element_index"] = element
args["window_id"] = self._active_window_id
elif x is not None and y is not None:
args["x"] = x
args["y"] = y
return self._action("scroll", args)
# ── Keyboard ───────────────────────────────────────────────────
def type_text(self, text: str) -> ActionResult:
pid = self._active_pid
if pid is None:
return ActionResult(ok=False, action="type_text",
message="No active window — call capture() first.")
return self._action("type_text", {"pid": pid, "text": text})
def key(self, keys: str) -> ActionResult:
pid = self._active_pid
if pid is None:
return ActionResult(ok=False, action="key",
message="No active window — call capture() first.")
key_name, modifiers = _parse_key_combo(keys)
if not key_name:
return ActionResult(ok=False, action="key",
message=f"Could not parse key from '{keys}'.")
if modifiers:
# hotkey requires at least one modifier + one key.
return self._action("hotkey", {"pid": pid, "keys": modifiers + [key_name]})
else:
return self._action("press_key", {"pid": pid, "key": key_name})
# ── Value setter ────────────────────────────────────────────────
def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
"""Set a value on an element. Handles AXPopUpButton selects natively."""
pid = self._active_pid
window_id = self._active_window_id
if pid is None or window_id is None:
return ActionResult(ok=False, action="set_value",
message="No active window — call capture() first.")
if element is None:
return ActionResult(ok=False, action="set_value",
message="set_value requires element= (element index).")
args: Dict[str, Any] = {
"pid": pid,
"window_id": window_id,
"element_index": element,
"value": value,
}
return self._action("set_value", args)
# ── Introspection ──────────────────────────────────────────────
def list_apps(self) -> List[Dict[str, Any]]:
out = self._session.call_tool("list_apps", {})
data = out["data"]
if isinstance(data, list):
return data
if isinstance(data, dict):
return data.get("apps", [])
# list_apps returns plain text — parse app lines.
if isinstance(data, str):
apps = []
for line in data.splitlines():
m = re.search(r'(.+?)\s+\(pid\s+(\d+)\)', line)
if m:
apps.append({"name": m.group(1).strip(), "pid": int(m.group(2))})
return apps
return []
def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
"""Target an app for subsequent actions without stealing system focus.
cua-driver background-automation never needs to bring a window to the
front: capture(app=...) already selects the right window via
list_windows. We implement focus_app as a pure window-selector —
enumerate on-screen windows, find the best match for *app*, and store
its pid/window_id so that subsequent click/type calls hit the right
process.
raise_window=True is intentionally ignored: stealing the user's focus
is exactly what this backend is designed to avoid.
"""
lw_out = self._session.call_tool("list_windows", {"on_screen_only": True})
sc = lw_out.get("structuredContent") or {}
raw_windows = sc.get("windows") if sc else None
if raw_windows:
windows = [
{
"app_name": w.get("app_name", ""),
"pid": int(w["pid"]),
"window_id": int(w["window_id"]),
"z_index": w.get("z_index", 0),
}
for w in raw_windows
]
windows.sort(key=lambda w: w["z_index"])
else:
raw_text = lw_out["data"] if isinstance(lw_out["data"], str) else ""
windows = _parse_windows_from_text(raw_text)
app_lower = app.lower()
matched = [w for w in windows if app_lower in w["app_name"].lower()]
# Don't silently fall back to the frontmost window when the filter
# matches nothing — that hides the real failure (often a localized
# macOS app name mismatch, e.g. caller passed "Calculator" but
# list_windows returns "計算機").
target = matched[0] if matched else None
if target:
self._active_pid = target["pid"]
self._active_window_id = target["window_id"]
self._last_app = target["app_name"] # preserve for capture_after= follow-ups
return ActionResult(
ok=True, action="focus_app",
message=f"Targeted {target['app_name']} (pid {self._active_pid}, "
f"window {self._active_window_id}) without raising window.",
)
return ActionResult(ok=False, action="focus_app",
message=f"No on-screen window found for app '{app}'.")
# ── Internal ───────────────────────────────────────────────────
def _action(self, name: str, args: Dict[str, Any]) -> ActionResult:
try:
out = self._session.call_tool(name, args)
except Exception as e:
logger.exception("cua-driver %s call failed", name)
return ActionResult(ok=False, action=name, message=f"cua-driver error: {e}")
ok = not out["isError"]
message = ""
data = out["data"]
if isinstance(data, dict):
message = str(data.get("message", ""))
elif isinstance(data, str):
message = data
return ActionResult(ok=ok, action=name, message=message,
meta=data if isinstance(data, dict) else {})
tools/computer_use/schema.py
+213
View File
@@ -0,0 +1,213 @@
"""Schema for the generic `computer_use` tool.
Model-agnostic. Any tool-calling model can drive this. Vision-capable models
should prefer `capture(mode='som')` then `click(element=N)` — much more
reliable than pixel coordinates. Pixel coordinates remain supported for
models that were trained on them (e.g. Claude's computer-use RL).
"""
from __future__ import annotations
from typing import Any, Dict
# One consolidated tool with an `action` discriminator. Keeps the schema
# compact and the per-turn token cost low.
COMPUTER_USE_SCHEMA: Dict[str, Any] = {
"name": "computer_use",
"description": (
"Drive the macOS desktop in the background — screenshots, mouse, "
"keyboard, scroll, drag — without stealing the user's cursor, "
"keyboard focus, or Space. Preferred workflow: call with "
"action='capture' (mode='som' gives numbered element overlays), "
"then click by `element` index for reliability. Pixel coordinates "
"are supported for models trained on them. Works on any window — "
"hidden, minimized, on another Space, or behind another app. "
"macOS only; requires cua-driver to be installed."
),
"parameters": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"capture",
"click",
"double_click",
"right_click",
"middle_click",
"drag",
"scroll",
"type",
"key",
"set_value",
"wait",
"list_apps",
"focus_app",
],
"description": (
"Which action to perform. `capture` is free (no side "
"effects). All other actions require approval unless "
"auto-approved. Use `set_value` for select/popup elements "
"and sliders — it selects the matching option directly "
"without opening the native menu (no focus steal)."
),
},
# ── capture ────────────────────────────────────────────
"mode": {
"type": "string",
"enum": ["som", "vision", "ax"],
"description": (
"Capture mode. `som` (default) is a screenshot with "
"numbered overlays on every interactable element plus "
"the AX tree — best for vision models, lets you click "
"by element index. `vision` is a plain screenshot. "
"`ax` is the accessibility tree only (no image; useful "
"for text-only models)."
),
},
"app": {
"type": "string",
"description": (
"Optional. Limit capture/action to a specific app "
"(by name, e.g. 'Safari', or bundle ID, "
"'com.apple.Safari'). If omitted, operates on the "
"frontmost app's window or the whole screen."
),
},
"max_elements": {
"type": "integer",
"description": (
"Optional cap on the AX `elements` array returned by "
"`action='capture'`. Default 100, hard maximum 1000. "
"Dense UIs (Electron apps such as Obsidian or VS Code, "
"JetBrains IDEs) can publish 500+ AX nodes — capping "
"prevents a single capture from blowing session "
"context. When the cap trims the response, "
"`total_elements` and `truncated_elements` are "
"surfaced in the result so you can re-call with "
"`app=` to narrow scope or raise `max_elements` when "
"the full tree is required. Has no effect on "
"`mode='som'` / `mode='vision'` when a screenshot is "
"included in the response; only the rare image-"
"missing fallback returns an `elements` array and is "
"subject to the cap."
),
"default": 100,
"minimum": 1,
"maximum": 1000,
},
# ── click / drag / scroll targeting ────────────────────
"element": {
"type": "integer",
"description": (
"The 1-based SOM index returned by the last "
"`capture(mode='som')` call. Strongly preferred over "
"raw coordinates."
),
},
"coordinate": {
"type": "array",
"items": {"type": "integer"},
"minItems": 2,
"maxItems": 2,
"description": (
"Pixel coordinates [x, y] in logical screen space (as "
"returned by capture width/height). Only use this if "
"no element index is available."
),
},
"button": {
"type": "string",
"enum": ["left", "right", "middle"],
"description": "Mouse button. Defaults to left.",
},
"modifiers": {
"type": "array",
"items": {
"type": "string",
"enum": ["cmd", "shift", "option", "alt", "ctrl", "fn"],
},
"description": "Modifier keys held during the action.",
},
# ── drag ───────────────────────────────────────────────
"from_element": {"type": "integer",
"description": "Source element index (drag)."},
"to_element": {"type": "integer",
"description": "Target element index (drag)."},
"from_coordinate": {
"type": "array",
"items": {"type": "integer"},
"minItems": 2, "maxItems": 2,
"description": "Source [x,y] (drag; use when no element available).",
},
"to_coordinate": {
"type": "array",
"items": {"type": "integer"},
"minItems": 2, "maxItems": 2,
"description": "Target [x,y] (drag; use when no element available).",
},
# ── scroll ─────────────────────────────────────────────
"direction": {
"type": "string",
"enum": ["up", "down", "left", "right"],
"description": "Scroll direction.",
},
"amount": {
"type": "integer",
"description": "Scroll wheel ticks. Default 3.",
},
# ── set_value ──────────────────────────────────────────
"value": {
"type": "string",
"description": (
"For action='set_value': the value to set on the element. "
"For AXPopUpButton / select dropdowns, pass the option's "
"display label (e.g. 'Blue'). For sliders and other "
"AXValue-settable elements, pass the numeric or string value."
),
},
# ── type / key / wait ──────────────────────────────────
"text": {
"type": "string",
"description": "Text to type (respects the current layout).",
},
"keys": {
"type": "string",
"description": (
"Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', "
"'escape', 'tab'. Use '+' to combine."
),
},
"seconds": {
"type": "number",
"description": "Seconds to wait. Max 30.",
},
# ── focus_app ──────────────────────────────────────────
"raise_window": {
"type": "boolean",
"description": (
"Only for action='focus_app'. If true, brings the "
"window to front (DISRUPTS the user). Default false "
"— input is routed to the app without raising, "
"matching the background co-work model."
),
},
# ── return shape ───────────────────────────────────────
"capture_after": {
"type": "boolean",
"description": (
"If true, take a follow-up capture after the action "
"and include it in the response. Saves a round-trip "
"when you need to verify an action's effect."
),
},
},
"required": ["action"],
},
}
def get_computer_use_schema() -> Dict[str, Any]:
"""Return the generic OpenAI function-calling schema."""
return COMPUTER_USE_SCHEMA
tools/computer_use/tool.py
+823
View File
@@ -0,0 +1,823 @@
"""Entry point for the `computer_use` tool.
Universal (any-model) macOS desktop control via cua-driver's background
computer-use primitive. Replaces #4562's Anthropic-native `computer_20251124`
approach — the schema here is standard OpenAI function-calling so every
tool-capable model can drive it.
Return contract
---------------
For text-only results (wait, key, list_apps, focus_app, failures, etc.):
JSON string.
For captures / actions with `capture_after=True`:
A dict wrapped as the OpenAI-style multi-part tool-message content:
{
"_multimodal": True,
"content": [
{"type": "text", "text": "<human-readable summary + SOM index>"},
{"type": "image_url",
"image_url": {"url": "data:image/png;base64,<b64>"}},
],
"text_summary": "<text used for fallback string content>",
}
run_agent.py's tool-message builder inspects `_multimodal` and emits a
list-shaped `content` for OpenAI-compatible providers. The Anthropic
adapter splices the base64 image into a `tool_result` block (see
`agent/anthropic_adapter.py`). Every provider that supports multi-part
tool content gets the image; text-only providers see the summary only.
"""
from __future__ import annotations
import base64
import json
import logging
import os
import re
import struct
import sys
import threading
from typing import Any, Dict, List, Optional, Tuple
from tools.computer_use.backend import (
ActionResult,
CaptureResult,
ComputerUseBackend,
UIElement,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Approval & safety
# ---------------------------------------------------------------------------
_approval_callback = None
def set_approval_callback(cb) -> None:
"""Register a callback for computer_use approval prompts (used by CLI).
Matches the terminal_tool._approval_callback pattern. The callback
receives (action, args, summary) and returns one of:
"approve_once" | "approve_session" | "always_approve" | "deny".
"""
global _approval_callback
_approval_callback = cb
# Actions that read, not mutate. Always allowed.
_SAFE_ACTIONS = frozenset({"capture", "wait", "list_apps"})
# Actions that mutate user-visible state. Go through approval.
_DESTRUCTIVE_ACTIONS = frozenset({
"click", "double_click", "right_click", "middle_click",
"drag", "scroll", "type", "key", "set_value", "focus_app",
})
# Hard-blocked key combinations. Mirrored from #4562 — these are destructive
# regardless of approval level (e.g. logout kills the session Hermes runs in).
_BLOCKED_KEY_COMBOS = {
frozenset({"cmd", "shift", "backspace"}), # empty trash
frozenset({"cmd", "option", "backspace"}), # force delete
frozenset({"cmd", "ctrl", "q"}), # lock screen
frozenset({"cmd", "shift", "q"}), # log out
frozenset({"cmd", "option", "shift", "q"}), # force log out
}
_KEY_ALIASES = {"command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option"}
def _canon_key_combo(keys: str) -> frozenset:
parts = [p.strip().lower() for p in re.split(r"\s*\+\s*", keys) if p.strip()]
parts = [_KEY_ALIASES.get(p, p) for p in parts]
return frozenset(parts)
# Dangerous text patterns for the `type` action. Same list as #4562.
_BLOCKED_TYPE_PATTERNS = [
re.compile(r"curl\s+[^|]*\|\s*bash", re.IGNORECASE),
re.compile(r"curl\s+[^|]*\|\s*sh", re.IGNORECASE),
re.compile(r"wget\s+[^|]*\|\s*bash", re.IGNORECASE),
re.compile(r"\bsudo\s+rm\s+-[rf]", re.IGNORECASE),
re.compile(r"\brm\s+-rf\s+/\s*$", re.IGNORECASE),
re.compile(r":\s*\(\)\s*\{\s*:\|:\s*&\s*\}", re.IGNORECASE), # fork bomb
]
def _is_blocked_type(text: str) -> Optional[str]:
for pat in _BLOCKED_TYPE_PATTERNS:
if pat.search(text):
return pat.pattern
return None
# ---------------------------------------------------------------------------
# Backend selection — env-swappable for tests
# ---------------------------------------------------------------------------
# Per-process cached backend; lazily instantiated on first call.
_backend_lock = threading.Lock()
_backend: Optional[ComputerUseBackend] = None
# Session-scoped approval state.
_session_auto_approve = False
_always_allow: set = set() # action names the user unlocked for the session
def _get_backend() -> ComputerUseBackend:
global _backend
with _backend_lock:
if _backend is None:
backend_name = os.environ.get("HERMES_COMPUTER_USE_BACKEND", "cua").lower()
if backend_name in {"cua", "cua-driver", ""}:
from tools.computer_use.cua_backend import CuaDriverBackend
_backend = CuaDriverBackend()
elif backend_name == "noop": # pragma: no cover
_backend = _NoopBackend()
else:
raise RuntimeError(f"Unknown HERMES_COMPUTER_USE_BACKEND={backend_name!r}")
_backend.start()
return _backend
def reset_backend_for_tests() -> None: # pragma: no cover
"""Test helper — tear down the cached backend."""
global _backend, _session_auto_approve, _always_allow
with _backend_lock:
if _backend is not None:
try:
_backend.stop()
except Exception:
pass
_backend = None
_session_auto_approve = False
_always_allow = set()
class _NoopBackend(ComputerUseBackend): # pragma: no cover
"""Test/CI stub. Records calls; returns trivial results."""
def __init__(self) -> None:
self.calls: List[Tuple[str, Dict[str, Any]]] = []
self._started = False
def start(self) -> None: self._started = True
def stop(self) -> None: self._started = False
def is_available(self) -> bool: return True
def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult:
self.calls.append(("capture", {"mode": mode, "app": app}))
return CaptureResult(mode=mode, width=1024, height=768, png_b64=None,
elements=[], app=app or "", window_title="")
def click(self, **kw) -> ActionResult:
self.calls.append(("click", kw))
return ActionResult(ok=True, action="click")
def drag(self, **kw) -> ActionResult:
self.calls.append(("drag", kw))
return ActionResult(ok=True, action="drag")
def scroll(self, **kw) -> ActionResult:
self.calls.append(("scroll", kw))
return ActionResult(ok=True, action="scroll")
def type_text(self, text: str) -> ActionResult:
self.calls.append(("type", {"text": text}))
return ActionResult(ok=True, action="type")
def key(self, keys: str) -> ActionResult:
self.calls.append(("key", {"keys": keys}))
return ActionResult(ok=True, action="key")
def list_apps(self) -> List[Dict[str, Any]]:
self.calls.append(("list_apps", {}))
return []
def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
self.calls.append(("focus_app", {"app": app, "raise": raise_window}))
return ActionResult(ok=True, action="focus_app")
def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
self.calls.append(("set_value", {"value": value, "element": element}))
return ActionResult(ok=True, action="set_value")
# ---------------------------------------------------------------------------
# Dispatch
# ---------------------------------------------------------------------------
def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any:
"""Main entry point — dispatched by tools.registry.
Returns either a JSON string (text-only) or a dict marked `_multimodal`
(image + summary) which run_agent.py wraps into the tool message.
"""
action = (args.get("action") or "").strip().lower()
if not action:
return json.dumps({"error": "missing `action`"})
# Safety: validate actions before approval prompt.
if action == "type":
text = args.get("text", "")
pat = _is_blocked_type(text)
if pat:
return json.dumps({
"error": f"blocked pattern in type text: {pat!r}",
"hint": "Dangerous shell patterns cannot be typed via computer_use.",
})
if action == "key":
keys = args.get("keys", "")
combo = _canon_key_combo(keys)
for blocked in _BLOCKED_KEY_COMBOS:
if blocked.issubset(combo) and len(blocked) <= len(combo):
return json.dumps({
"error": f"blocked key combo: {sorted(blocked)}",
"hint": "Destructive system shortcuts are hard-blocked.",
})
# Approval gate (destructive actions only).
if action in _DESTRUCTIVE_ACTIONS:
err = _request_approval(action, args)
if err is not None:
return err
# Dispatch to backend.
try:
backend = _get_backend()
except Exception as e:
return json.dumps({
"error": f"computer_use backend unavailable: {e}",
"hint": "Run `hermes tools` and enable Computer Use to install cua-driver.",
})
try:
return _dispatch(backend, action, args)
except Exception as e:
logger.exception("computer_use %s failed", action)
return json.dumps({"error": f"{action} failed: {e}"})
def _request_approval(action: str, args: Dict[str, Any]) -> Optional[str]:
"""Return None if approved, or a JSON error string if denied."""
global _session_auto_approve, _always_allow
if _session_auto_approve:
return None
if action in _always_allow:
return None
cb = _approval_callback
if cb is None:
# No CLI approval wired — default allow. Gateway approval is handled
# one layer out via the normal tool-approval infra.
return None
summary = _summarize_action(action, args)
try:
verdict = cb(action, args, summary)
except Exception as e:
logger.warning("approval callback failed: %s", e)
verdict = "deny"
if verdict == "approve_once":
return None
if verdict == "approve_session" or verdict == "always_approve":
_always_allow.add(action)
if verdict == "always_approve":
_session_auto_approve = True
return None
return json.dumps({"error": "denied by user", "action": action})
def _summarize_action(action: str, args: Dict[str, Any]) -> str:
if action in {"click", "double_click", "right_click", "middle_click"}:
if args.get("element") is not None:
return f"{action} element #{args['element']}"
coord = args.get("coordinate")
if coord:
return f"{action} at {tuple(coord)}"
return action
if action == "drag":
src = args.get("from_element") or args.get("from_coordinate")
dst = args.get("to_element") or args.get("to_coordinate")
return f"drag {src} → {dst}"
if action == "scroll":
return f"scroll {args.get('direction', '?')} x{args.get('amount', 3)}"
if action == "type":
text = args.get("text", "")
return f"type {text[:60]!r}" + ("..." if len(text) > 60 else "")
if action == "key":
return f"key {args.get('keys', '')!r}"
if action == "focus_app":
return f"focus {args.get('app', '')!r}" + (" (raise)" if args.get("raise_window") else "")
return action
def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> Any:
capture_after = bool(args.get("capture_after"))
if action == "capture":
mode = str(args.get("mode", "som"))
if mode not in {"som", "vision", "ax"}:
return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"})
cap = backend.capture(mode=mode, app=args.get("app"))
return _capture_response(cap, max_elements=_coerce_max_elements(args.get("max_elements")))
if action == "wait":
seconds = float(args.get("seconds", 1.0))
res = backend.wait(seconds)
return _text_response(res)
if action == "list_apps":
apps = backend.list_apps()
return json.dumps({"apps": apps, "count": len(apps)})
if action == "focus_app":
app = args.get("app")
if not app:
return json.dumps({"error": "focus_app requires `app`"})
res = backend.focus_app(app, raise_window=bool(args.get("raise_window")))
return _maybe_follow_capture(backend, res, capture_after)
if action in {"click", "double_click", "right_click", "middle_click"}:
button = args.get("button")
click_count = 1
if action == "double_click":
click_count = 2
elif action == "right_click":
button = "right"
elif action == "middle_click":
button = "middle"
else:
button = button or "left"
element = args.get("element")
coord = args.get("coordinate") or (None, None)
x, y = (coord[0], coord[1]) if coord and coord[0] is not None else (None, None)
res = backend.click(
element=element if element is not None else None,
x=x, y=y, button=button or "left", click_count=click_count,
modifiers=args.get("modifiers"),
)
return _maybe_follow_capture(backend, res, capture_after)
if action == "drag":
has_elements = args.get("from_element") is not None and args.get("to_element") is not None
has_coords = args.get("from_coordinate") and args.get("to_coordinate")
if not has_elements and not has_coords:
return json.dumps({
"error": "drag requires from_coordinate/to_coordinate or from_element/to_element",
})
res = backend.drag(
from_element=args.get("from_element"),
to_element=args.get("to_element"),
from_xy=tuple(args["from_coordinate"]) if args.get("from_coordinate") else None,
to_xy=tuple(args["to_coordinate"]) if args.get("to_coordinate") else None,
button=args.get("button", "left"),
modifiers=args.get("modifiers"),
)
return _maybe_follow_capture(backend, res, capture_after)
if action == "scroll":
coord = args.get("coordinate") or (None, None)
res = backend.scroll(
direction=args.get("direction", "down"),
amount=int(args.get("amount", 3)),
element=args.get("element"),
x=coord[0] if coord and coord[0] is not None else None,
y=coord[1] if coord and coord[1] is not None else None,
modifiers=args.get("modifiers"),
)
return _maybe_follow_capture(backend, res, capture_after)
if action == "type":
res = backend.type_text(args.get("text", ""))
return _maybe_follow_capture(backend, res, capture_after)
if action == "key":
res = backend.key(args.get("keys", ""))
return _maybe_follow_capture(backend, res, capture_after)
if action == "set_value":
value = args.get("value")
if value is None:
return json.dumps({"error": "set_value requires `value`"})
res = backend.set_value(value=str(value), element=args.get("element"))
return _maybe_follow_capture(backend, res, capture_after)
return json.dumps({"error": f"unknown action {action!r}"})
# ---------------------------------------------------------------------------
# Response shaping
# ---------------------------------------------------------------------------
def _text_response(res: ActionResult) -> str:
payload: Dict[str, Any] = {"ok": res.ok, "action": res.action}
if res.message:
payload["message"] = res.message
if res.meta:
payload["meta"] = res.meta
return json.dumps(payload)
# Default cap for the AX `elements` array returned by capture. Dense UIs
# (Electron apps, Obsidian, JetBrains IDEs) can publish 500+ AX nodes, which
# can exhaust session context after a single capture. The model-facing
# `max_elements` argument lets callers raise this when they need the full tree.
_DEFAULT_MAX_ELEMENTS = 100
# Hard upper bound on caller-supplied `max_elements`. Without this, a tool
# call passing a very large integer would silently disable the safeguard and
# reintroduce the original unbounded behavior.
_MAX_ALLOWED_MAX_ELEMENTS = 1000
_MIN_PROVIDER_IMAGE_DIMENSION = 8
def _image_dimensions_from_b64(image_b64: str) -> Optional[Tuple[int, int]]:
"""Return (width, height) for common inline screenshot formats.
Some providers reject images below 8x8 before the model sees the tool
result. Inspecting the encoded bytes here lets computer_use fall back to
its AX/SOM text payload instead of sending an unusable placeholder.
"""
if not image_b64:
return None
try:
raw = base64.b64decode(image_b64, validate=False)
except Exception:
return None
# PNG: signature + IHDR width/height.
if raw.startswith(b"\x89PNG\r\n\x1a\n") and len(raw) >= 24:
try:
width, height = struct.unpack(">II", raw[16:24])
return int(width), int(height)
except Exception:
return None
# JPEG: scan for SOF markers that carry dimensions.
if raw.startswith(b"\xff\xd8") and len(raw) > 4:
i = 2
while i + 9 < len(raw):
if raw[i] != 0xFF:
i += 1
continue
marker = raw[i + 1]
i += 2
while marker == 0xFF and i < len(raw):
marker = raw[i]
i += 1
if marker in {0xD8, 0xD9}:
continue
if marker == 0xDA:
break
if i + 2 > len(raw):
break
segment_len = int.from_bytes(raw[i:i + 2], "big")
if segment_len < 2 or i + segment_len > len(raw):
break
if marker in {
0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7,
0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF,
} and segment_len >= 7:
height = int.from_bytes(raw[i + 3:i + 5], "big")
width = int.from_bytes(raw[i + 5:i + 7], "big")
return int(width), int(height)
i += segment_len
return None
def _coerce_max_elements(value: Any) -> int:
"""Validate the caller-supplied ``max_elements``.
Falls back to :data:`_DEFAULT_MAX_ELEMENTS` for missing / non-integer /
sub-1 inputs so the cap can never be silently disabled by a malformed
tool-call argument. Clamps oversized values to
:data:`_MAX_ALLOWED_MAX_ELEMENTS` so a caller cannot bypass the
safeguard by passing a very large integer.
"""
if value is None:
return _DEFAULT_MAX_ELEMENTS
try:
n = int(value)
except (TypeError, ValueError):
return _DEFAULT_MAX_ELEMENTS
if n < 1:
return _DEFAULT_MAX_ELEMENTS
if n > _MAX_ALLOWED_MAX_ELEMENTS:
return _MAX_ALLOWED_MAX_ELEMENTS
return n
def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEMENTS) -> Any:
total_elements = len(cap.elements)
visible_elements = cap.elements[:max_elements]
truncated_elements = max(0, total_elements - len(visible_elements))
image_dimensions = _image_dimensions_from_b64(cap.png_b64 or "") if cap.png_b64 else None
response_width = image_dimensions[0] if image_dimensions else cap.width
response_height = image_dimensions[1] if image_dimensions else cap.height
image_too_small = bool(
image_dimensions
and (
image_dimensions[0] < _MIN_PROVIDER_IMAGE_DIMENSION
or image_dimensions[1] < _MIN_PROVIDER_IMAGE_DIMENSION
)
)
# Index only what's actually surfaced in the response — otherwise the
# human-readable summary references element indices the model cannot
# find in the JSON `elements` array (e.g. max_elements=10 vs the default
# 40-line index window).
element_index = _format_elements(visible_elements)
summary_lines = [
f"capture mode={cap.mode} {response_width}x{response_height}"
+ (f" app={cap.app}" if cap.app else "")
+ (f" window={cap.window_title!r}" if cap.window_title else ""),
f"{total_elements} interactable element(s):",
]
if element_index:
summary_lines.extend(element_index)
# Multimodal and AX paths both reference `summary`; build it once up-front
# so the aux-vision routing branch (which fires before either path is
# selected) has a valid value to hand to _route_capture_through_aux_vision.
# The AX path appends the "truncated to N of M" note to summary_lines
# below and rebuilds; the multimodal path keeps this version untouched.
if image_too_small:
summary_lines.append(
f" (screenshot omitted: {image_dimensions[0]}x{image_dimensions[1]} "
f"is below the {_MIN_PROVIDER_IMAGE_DIMENSION}x{_MIN_PROVIDER_IMAGE_DIMENSION} "
"provider minimum)"
)
summary = "\n".join(summary_lines)
if cap.png_b64 and cap.mode != "ax" and not image_too_small:
# Decide whether to hand the screenshot to the auxiliary.vision
# pipeline (text-only result) or keep the multimodal envelope (main
# model handles vision natively). Issue #24015: previously the
# multimodal envelope was returned unconditionally, so non-vision
# main models tripped HTTP 404 / 400 at the provider boundary even
# when auxiliary.vision was explicitly configured to handle this.
if _should_route_through_aux_vision():
routed = _route_capture_through_aux_vision(cap, summary)
if routed is not None:
return routed
# Aux routing was requested but failed (no vision client, aux
# call raised, etc.). Fall through to the multimodal envelope —
# better to surface a tool-result error from the main model
# than to silently drop the screenshot entirely.
# Detect actual image format from base64 magic bytes so the MIME type
# matches what the data contains (cua-driver may return JPEG or PNG).
# JPEG: base64 starts with /9j/ PNG: starts with iVBOR
_b64_prefix = cap.png_b64[:8]
_mime = "image/jpeg" if _b64_prefix.startswith("/9j/") else "image/png"
# The multimodal response carries the screenshot, not the AX
# elements array, so a "response truncated to N of M elements"
# note would be inaccurate — skip it on this branch.
return {
"_multimodal": True,
"content": [
{"type": "text", "text": summary},
{"type": "image_url",
"image_url": {"url": f"data:{_mime};base64,{cap.png_b64}"}},
],
"text_summary": summary,
"meta": {"mode": cap.mode, "width": response_width, "height": response_height,
"elements": total_elements, "png_bytes": cap.png_bytes_len},
}
# AX-only (or image-missing fallback): text path actually carries the
# `elements` array, so the truncation note applies here.
if truncated_elements:
summary_lines.append(
f" (response truncated to {len(visible_elements)} of {total_elements} elements; "
f"raise max_elements or pass app= to narrow)"
)
summary = "\n".join(summary_lines)
payload: Dict[str, Any] = {
"mode": cap.mode,
"width": response_width,
"height": response_height,
"app": cap.app,
"window_title": cap.window_title,
"elements": [_element_to_dict(e) for e in visible_elements],
"total_elements": total_elements,
"summary": summary,
}
if truncated_elements:
payload["truncated_elements"] = truncated_elements
return json.dumps(payload)
# ---------------------------------------------------------------------------
# auxiliary.vision routing for captured screenshots (#24015)
# ---------------------------------------------------------------------------
def _should_route_through_aux_vision() -> bool:
"""Return True when ``_capture_response`` should hand the PNG to aux vision.
Reads the active main provider/model and the loaded config and asks the
routing helper. Any failure (config import, runtime override missing,
etc.) returns False so the existing multimodal envelope continues to be
returned — fail open on the routing decision so a broken config can
never silently drop the screenshot for vision-capable main models.
"""
try:
from agent.auxiliary_client import _read_main_model, _read_main_provider
from hermes_cli.config import load_config
from tools.computer_use.vision_routing import (
should_route_capture_to_aux_vision,
)
except Exception as exc: # pragma: no cover - defensive
logger.debug("computer_use: aux-vision routing import failed: %s", exc)
return False
try:
provider = _read_main_provider()
model = _read_main_model()
cfg = load_config()
except Exception as exc: # pragma: no cover - defensive
logger.debug("computer_use: aux-vision routing config read failed: %s", exc)
return False
try:
return bool(should_route_capture_to_aux_vision(provider, model, cfg))
except Exception as exc: # pragma: no cover - defensive
logger.debug("computer_use: aux-vision routing decision failed: %s", exc)
return False
def _route_capture_through_aux_vision(
cap: CaptureResult,
summary: str,
) -> Optional[str]:
"""Pre-analyse the captured PNG via ``vision_analyze`` and return a text result.
The captured base64 PNG is materialised to ``$HERMES_HOME/cache/vision/``
and handed to ``vision_analyze_tool`` with a generic describe prompt.
The resulting text description is merged into the existing AX/SOM
summary so the main model receives a single text payload that mentions
every interactable element AND a description of what the screenshot
looked like.
Returns:
A JSON-encoded text response on success.
``None`` on failure (caller falls back to the multimodal envelope).
"""
if not cap.png_b64:
return None
try:
import base64 as _base64
import os as _os
import uuid as _uuid
from hermes_constants import get_hermes_dir
from model_tools import _run_async
from tools.vision_tools import vision_analyze_tool
except Exception as exc: # pragma: no cover - defensive
logger.debug("computer_use: aux-vision import failed: %s", exc)
return None
temp_image_path = None
try:
try:
raw = _base64.b64decode(cap.png_b64, validate=False)
except Exception as exc:
logger.debug("computer_use: failed to decode capture base64: %s", exc)
return None
# Pick an extension that matches the on-disk bytes so vision_analyze's
# MIME sniffing returns the right content-type.
ext = ".jpg" if cap.png_b64[:8].startswith("/9j/") else ".png"
cache_dir = get_hermes_dir("cache/vision", "temp_vision_images")
cache_dir.mkdir(parents=True, exist_ok=True)
temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}"
temp_image_path.write_bytes(raw)
prompt = (
"Describe what is visible in this macOS application screenshot in "
"concise but specific terms. Mention the app name and window "
"title if visible, the overall layout, any labelled buttons, "
"menus or text fields, and any prominent text content the user "
"would need to know about. Do not invent details that are not "
"actually visible.\n\n"
f"AX/SOM index for cross-reference:\n{summary}"
)
result_json = _run_async(
vision_analyze_tool(str(temp_image_path), prompt)
)
except Exception as exc:
logger.warning(
"computer_use: auxiliary.vision pre-analysis failed (%s); "
"falling back to native multimodal envelope",
exc,
)
return None
finally:
if temp_image_path is not None:
try:
_os.unlink(str(temp_image_path))
except Exception:
pass
analysis_text = ""
if isinstance(result_json, str):
try:
parsed = json.loads(result_json)
if isinstance(parsed, dict):
analysis_text = str(parsed.get("analysis") or "").strip()
except (TypeError, json.JSONDecodeError):
analysis_text = result_json.strip()
if not analysis_text:
return None
return json.dumps({
"mode": cap.mode,
"width": cap.width,
"height": cap.height,
"app": cap.app,
"window_title": cap.window_title,
"elements": [_element_to_dict(e) for e in cap.elements],
"summary": summary,
"vision_analysis": analysis_text,
"vision_analysis_routed_via": "auxiliary.vision",
})
def _maybe_follow_capture(
backend: ComputerUseBackend, res: ActionResult, do_capture: bool,
) -> Any:
if not do_capture:
return _text_response(res)
# Skip the follow-up capture when the action itself failed: showing a
# normal-looking screenshot after a failure misleads the model into thinking
# the action succeeded. Return the error text instead.
if not res.ok:
return _text_response(res)
try:
# Preserve the app context established by the preceding capture/focus_app so
# that capture_after=True re-captures the same app rather than the frontmost
# window (which may have changed if the action caused a focus shift).
last_app = getattr(backend, "_last_app", None)
cap = backend.capture(mode="som", app=last_app)
except Exception as e:
logger.warning("follow-up capture failed: %s", e)
return _text_response(res)
# Combine action summary with the capture.
resp = _capture_response(cap)
if isinstance(resp, dict) and resp.get("_multimodal"):
prefix = f"[{res.action}] ok={res.ok}" + (f" — {res.message}" if res.message else "")
resp["content"][0]["text"] = prefix + "\n\n" + resp["content"][0]["text"]
resp["text_summary"] = prefix + "\n\n" + resp["text_summary"]
return resp
# Fallback: action + text capture merged.
try:
data = json.loads(resp)
except (TypeError, json.JSONDecodeError):
data = {"capture": resp}
data["action"] = res.action
data["ok"] = res.ok
if res.message:
data["message"] = res.message
return json.dumps(data)
def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str]:
out: List[str] = []
for e in elements[:max_lines]:
label = e.label.replace("\n", " ")[:60]
out.append(f" #{e.index} {e.role} {label!r} @ {e.bounds}"
+ (f" [{e.app}]" if e.app else ""))
if len(elements) > max_lines:
out.append(f" ... +{len(elements) - max_lines} more (call capture with app= to narrow)")
return out
def _element_to_dict(e: UIElement) -> Dict[str, Any]:
return {
"index": e.index,
"role": e.role,
"label": e.label,
"bounds": list(e.bounds),
"app": e.app,
}
# ---------------------------------------------------------------------------
# Availability check (used by the tool registry check_fn)
# ---------------------------------------------------------------------------
def check_computer_use_requirements() -> bool:
"""Return True iff computer_use can run on this host.
Conditions: macOS + cua-driver binary installed (or override via env).
"""
if sys.platform != "darwin":
return False
from tools.computer_use.cua_backend import cua_driver_binary_available
return cua_driver_binary_available()
def get_computer_use_schema() -> Dict[str, Any]:
from tools.computer_use.schema import COMPUTER_USE_SCHEMA
return COMPUTER_USE_SCHEMA
tools/computer_use/vision_routing.py
+204
View File
@@ -0,0 +1,204 @@
"""Vision-routing decisions for ``computer_use`` capture results.
Background
----------
``computer_use(action='capture', mode='som'|'vision')`` returns a
``_multimodal`` envelope containing the captured screenshot. That envelope
is delivered back to the **active session model** as the tool result. When
the active main model has no vision capability (e.g. text-only or
text+code-only models), or when the active provider rejects multimodal
content inside tool-result messages, the screenshot trips a 404 / 400 at
the provider boundary and the agent loop reports a hard tool failure.
Issue #24015 reports this regression for the ``cua-driver`` backend:
configuring ``auxiliary.vision`` (a dedicated vision-capable model) in
``config.yaml`` was silently ignored — the screenshot was still routed at
the *main* model and failed with HTTP 404 ``No endpoints found that
support image input`` even though a perfectly good vision backend was
sitting in config waiting to be used.
This module centralises the small policy decision: should a captured
screenshot be returned as multimodal content (main model handles vision
natively) or pre-analysed via the auxiliary vision pipeline so the main
model only ever sees text?
Behaviour (mirrors ``vision_analyze`` for consistency)
------------------------------------------------------
* If the user explicitly configured ``auxiliary.vision`` (any of
``provider``, ``model``, or ``base_url`` non-empty / not ``"auto"``),
the screenshot is routed through the aux vision pipeline. Users who
pay for a dedicated vision model usually want it used.
* Otherwise, if the user explicitly declared the active model vision-capable
via ``model.supports_vision`` / provider model config, return ``False``.
This is the escape hatch for custom/local OpenAI-compatible VLM routes that
are absent from models.dev and provider allowlists.
* Otherwise, if the active main model+provider can carry an image inside
a tool-result message AND the model reports ``supports_vision=True``
in models.dev metadata, return ``False`` (use the multimodal path).
* In every other case (non-vision main model, provider that does not
accept multimodal tool results, lookup failure), route through aux
vision so the main model receives a text description it can act on.
The decision intentionally fails *closed* (i.e. towards aux routing) when
metadata is missing or ambiguous: returning a screenshot to a model that
cannot read it is a hard tool failure, while routing it through aux costs
one extra LLM call and yields a usable description.
"""
from __future__ import annotations
import logging
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:
"""True when ``auxiliary.vision`` carries a non-default user override.
Mirrors ``agent.image_routing._explicit_aux_vision_override`` so the
capture path and the user-attached-image path agree on what counts as
an explicit user request for the aux vision pipeline. ``provider:
"auto"``, blank values, or a missing block all count as *not*
explicit.
"""
if not isinstance(cfg, dict):
return False
aux = cfg.get("auxiliary") or {}
if not isinstance(aux, dict):
return False
vision = aux.get("vision") or {}
if not isinstance(vision, dict):
return False
provider = str(vision.get("provider") or "").strip().lower()
model = str(vision.get("model") or "").strip()
base_url = str(vision.get("base_url") or "").strip()
if provider in ("", "auto") and not model and not base_url:
return False
return True
def _lookup_user_declared_supports_vision(
provider: str,
model: str,
cfg: Optional[Dict[str, Any]],
) -> Optional[bool]:
"""Return config-declared ``supports_vision`` for the active route."""
try:
from agent.image_routing import _supports_vision_override
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: config override lookup import failed: %s",
exc,
)
return None
try:
return _supports_vision_override(cfg, provider, model)
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: config override lookup failed: %s",
exc,
)
return None
def _lookup_supports_vision(
provider: str,
model: str,
cfg: Optional[Dict[str, Any]] = None,
) -> Optional[bool]:
"""Return config/models.dev ``supports_vision`` for *(provider, model)*."""
if not provider or not model:
return None
try:
from agent.image_routing import _lookup_supports_vision as _lookup_image_supports
except Exception:
_lookup_image_supports = None
if _lookup_image_supports is not None:
try:
return _lookup_image_supports(provider, model, cfg)
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: image-routing caps lookup failed "
"for %s:%s — %s",
provider, model, exc,
)
return None
try:
from agent.models_dev import get_model_capabilities
caps = get_model_capabilities(provider, model)
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: caps lookup failed for %s:%s — %s",
provider, model, exc,
)
return None
if caps is None:
return None
return bool(getattr(caps, "supports_vision", False))
def _provider_accepts_multimodal_tool_result(provider: str, model: str) -> Optional[bool]:
"""Return whether *provider*+*model* carries images inside tool-result messages.
Reuses ``tools.vision_tools._supports_media_in_tool_results`` so the
capture-routing decision stays in lockstep with the
``vision_analyze`` native fast path. Returns None on import failure
so callers fall back to aux routing rather than guessing.
"""
if not provider:
return None
try:
from tools.vision_tools import _supports_media_in_tool_results
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: tool-result support lookup failed: %s",
exc,
)
return None
return bool(_supports_media_in_tool_results(provider, model))
def should_route_capture_to_aux_vision(
provider: str,
model: str,
cfg: Optional[Dict[str, Any]],
) -> bool:
"""Return True iff the captured screenshot should be pre-analysed via aux vision.
Args:
provider: active inference provider id (e.g. ``"openrouter"``,
``"anthropic"``, ``"openai-codex"``). Lower-case canonical id.
model: active main model slug as it would be sent to the provider.
cfg: loaded ``config.yaml`` dict (or None).
Returns:
``True`` when the caller should hand the screenshot to the aux vision
pipeline (and surface a text-only tool result). ``False`` when the
caller should keep the existing multimodal envelope (main model
handles vision natively).
"""
if _explicit_aux_vision_override(cfg):
return True
user_declared = _lookup_user_declared_supports_vision(provider, model, cfg)
if user_declared is True:
return False
if user_declared is False:
return True
accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)
if accepts_tool_image is None or accepts_tool_image is False:
return True
supports_vision = _lookup_supports_vision(provider, model, cfg)
if supports_vision is True:
return False
return True
__all__ = [
"should_route_capture_to_aux_vision",
]