feat(browser): integrate Browser Use CLI 3.0
This commit is contained in:
Laith Weinberger
Teknium
parent
eb4a0a3da7
commit
a1835c8c17
|
|
@ -39,7 +39,7 @@ which provider is in use.
|
|||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
from typing import Any, Dict
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -126,7 +126,7 @@ class BrowserProvider(abc.ABC):
|
|||
credentials, network errors, etc. — log and move on. Must not raise.
|
||||
"""
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
def get_setup_schema(self) -> Optional[Dict[str, Any]]:
|
||||
"""Return provider metadata for the ``hermes tools`` picker.
|
||||
|
||||
Used by :mod:`hermes_cli.tools_config` to inject this provider as a
|
||||
|
|
|
|||
|
|
@ -427,6 +427,22 @@ def _delegate_task_goal_parts(tasks: Any, *, per_goal_len: int) -> tuple[int, li
|
|||
return len(goals), goals
|
||||
|
||||
|
||||
def _browser_exec_step_label(args: dict, max_chars: int = 80) -> str | None:
|
||||
"""User-friendly step label from browser_exec code's leading comment."""
|
||||
code = str(args.get("code", "") or "").strip()
|
||||
if not code:
|
||||
return None
|
||||
first = code.split("\n", 1)[0].strip()
|
||||
if not first.startswith("#"):
|
||||
return None
|
||||
label = first.lstrip("#").strip()
|
||||
if not label:
|
||||
return None
|
||||
if len(label) > max_chars:
|
||||
label = label[: max_chars - 1] + "…"
|
||||
return label
|
||||
|
||||
|
||||
def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) -> str | None:
|
||||
"""Build a short preview of a tool call's primary argument for display.
|
||||
|
||||
|
|
@ -447,10 +463,18 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) -
|
|||
"vision_analyze": "question",
|
||||
"skill_view": "name", "skills_list": "category",
|
||||
"cronjob": "action",
|
||||
"execute_code": "code", "delegate_task": "goal",
|
||||
"execute_code": "code", "browser_exec": "code", "delegate_task": "goal",
|
||||
"clarify": "question", "skill_manage": "name",
|
||||
}
|
||||
|
||||
# browser_exec: prefer the leading `# …` comment as a friendly step label
|
||||
if tool_name == "browser_exec":
|
||||
label = _browser_exec_step_label(args)
|
||||
if label is not None:
|
||||
return _truncate_preview(label, max_len)
|
||||
preview = _oneline(str(args.get("code", "") or ""))
|
||||
return _truncate_preview(preview, max_len) if preview else None
|
||||
|
||||
# delegate_task: show goal (single) or individual task goals (batch)
|
||||
if tool_name == "delegate_task":
|
||||
tasks = args.get("tasks")
|
||||
|
|
@ -1516,6 +1540,15 @@ def _get_cute_tool_message(
|
|||
code = args.get("code", "")
|
||||
first_line = code.strip().split("\n")[0] if code.strip() else ""
|
||||
return _wrap(f"┊ 🐍 exec {_trunc(first_line, 35)} {dur}")
|
||||
if tool_name == "browser_exec":
|
||||
label = _browser_exec_step_label(args)
|
||||
if label is not None:
|
||||
# Leading `# …` comment (the tool description asks for one):
|
||||
# surface it as the user-facing step label; the code itself stays
|
||||
# collapsed behind display.tool_preview_length.
|
||||
return _wrap(f"┊ 🌐 browser {label} {dur}")
|
||||
code = " ".join(str(args.get("code", "") or "").split())
|
||||
return _wrap(f"┊ 🌐 browser {_trunc(code, 35)} {dur}")
|
||||
if tool_name == "delegate_task":
|
||||
tasks = args.get("tasks")
|
||||
if tasks and isinstance(tasks, list):
|
||||
|
|
|
|||
|
|
@ -2187,6 +2187,52 @@ class CLICommandsMixin:
|
|||
_DEFAULT_CDP = DEFAULT_BROWSER_CDP_URL
|
||||
current = os.environ.get("BROWSER_CDP_URL", "").strip()
|
||||
|
||||
if sub == "use" or sub.startswith("use "):
|
||||
# /browser use [off] — toggle Browser Use mode (browser.backend),
|
||||
arg = sub.split(None, 1)[1].strip() if " " in sub else "on"
|
||||
from hermes_cli.config import load_config, save_config
|
||||
from tools.registry import invalidate_check_fn_cache
|
||||
|
||||
if arg not in {"on", "off"}:
|
||||
print()
|
||||
print("Usage: /browser use [off]")
|
||||
print(" /browser use — switch to Browser Use mode (browser_exec via CLI 3.0)")
|
||||
print(" /browser use off — revert to the built-in browser tools")
|
||||
print()
|
||||
return
|
||||
|
||||
config = load_config()
|
||||
browser_cfg = config.setdefault("browser", {})
|
||||
if arg == "on":
|
||||
browser_cfg["backend"] = "browser-use"
|
||||
save_config(config)
|
||||
invalidate_check_fn_cache()
|
||||
self.new_session()
|
||||
print()
|
||||
print("🌐 Browser Use mode enabled — browser_exec via the Browser Use CLI 3.0")
|
||||
print(" Session reset. New tool configuration is active.")
|
||||
print()
|
||||
else:
|
||||
browser_cfg.pop("backend", None)
|
||||
save_config(config)
|
||||
invalidate_check_fn_cache()
|
||||
self.new_session()
|
||||
print()
|
||||
print("🌐 Browser Use mode disabled — built-in browser tools restored")
|
||||
try:
|
||||
from tools.browser_use_cli import is_browser_use_cli_mode
|
||||
if is_browser_use_cli_mode():
|
||||
print(
|
||||
" ⚠ Still active via auto-detection: BROWSER_USE_API_KEY is set "
|
||||
"with no other cloud provider configured."
|
||||
)
|
||||
print(" Pick another provider via `hermes tools`, or unset the key.")
|
||||
except Exception:
|
||||
pass
|
||||
print(" Session reset. New tool configuration is active.")
|
||||
print()
|
||||
return
|
||||
|
||||
if sub.startswith("connect"):
|
||||
# Optionally accept a custom CDP URL: /browser connect ws://host:port
|
||||
connect_parts = cmd.strip().split(None, 2) # ["/browser", "connect", "ws://..."]
|
||||
|
|
@ -2350,6 +2396,18 @@ class CLICommandsMixin:
|
|||
|
||||
elif sub == "status":
|
||||
print()
|
||||
try:
|
||||
from tools.browser_use_cli import is_browser_use_cli_mode
|
||||
_bu_mode = is_browser_use_cli_mode()
|
||||
except Exception:
|
||||
_bu_mode = False
|
||||
if _bu_mode:
|
||||
print("🌐 Browser: Browser Use mode (browser_exec via the Browser Use CLI 3.0)")
|
||||
print(" Local Chrome via CDP, or Browser Use cloud browsers")
|
||||
print()
|
||||
print(" /browser use off — revert to the built-in browser tools")
|
||||
print()
|
||||
return
|
||||
if current:
|
||||
print("🌐 Browser: connected to live Chromium-family browser via CDP")
|
||||
print(f" Endpoint: {current}")
|
||||
|
|
@ -2399,11 +2457,12 @@ class CLICommandsMixin:
|
|||
|
||||
else:
|
||||
print()
|
||||
print("Usage: /browser connect|disconnect|status")
|
||||
print("Usage: /browser connect|disconnect|status|use")
|
||||
print()
|
||||
print(" connect Connect browser tools to your live Chromium-family browser session")
|
||||
print(" disconnect Revert to default browser backend")
|
||||
print(" status Show current browser mode")
|
||||
print(" use [off] Switch to Browser Use mode (CLI 3.0) / back to built-in tools")
|
||||
print()
|
||||
|
||||
def _handle_heartbeat_command(self, cmd: str) -> None:
|
||||
|
|
|
|||
|
|
@ -301,9 +301,9 @@ COMMAND_REGISTRY: list[CommandDef] = [
|
|||
aliases=("reload_mcp",)),
|
||||
CommandDef("reload-skills", "Re-scan ~/.hermes/skills/ for newly installed or removed skills",
|
||||
"Tools & Skills", aliases=("reload_skills",)),
|
||||
CommandDef("browser", "Connect browser tools to your live Chromium-family browser via CDP", "Tools & Skills",
|
||||
cli_only=True, args_hint="[connect|disconnect|status]",
|
||||
subcommands=("connect", "disconnect", "status")),
|
||||
CommandDef("browser", "Connect browser tools to your live Chromium-family browser via CDP, or switch to Browser Use mode", "Tools & Skills",
|
||||
cli_only=True, args_hint="[connect|disconnect|status|use]",
|
||||
subcommands=("connect", "disconnect", "status", "use")),
|
||||
CommandDef("plugins", "List installed plugins and their status",
|
||||
"Tools & Skills", cli_only=True),
|
||||
|
||||
|
|
|
|||
|
|
@ -403,6 +403,12 @@ DEFAULT_CONFIG = {
|
|||
},
|
||||
|
||||
"browser": {
|
||||
# Browser tool implementation.
|
||||
# "" — built-in browser tools (browser_navigate, browser_click, …)
|
||||
# "browser-use" — Browser Use mode: one browser_exec tool driving the
|
||||
# Browser Use CLI 3.0 (local Chrome over CDP or Browser
|
||||
# Use cloud browsers)
|
||||
"backend": "",
|
||||
"inactivity_timeout": 120,
|
||||
"command_timeout": 30, # Timeout for browser commands in seconds (screenshot, navigate, etc.)
|
||||
"record_sessions": False, # Auto-record browser sessions as WebM videos
|
||||
|
|
|
|||
|
|
@ -622,6 +622,7 @@ TOOL_CATEGORIES = {
|
|||
# underlying backend but has a distinct setup UX.
|
||||
# - "Camofox" — anti-detection local Firefox; short-circuits the
|
||||
# cloud-provider dispatch path via _is_camofox_mode().
|
||||
# - "Browser Use" — the Browser Use CLI 3.0
|
||||
"providers": [
|
||||
{
|
||||
"name": "Local Browser",
|
||||
|
|
@ -658,6 +659,14 @@ TOOL_CATEGORIES = {
|
|||
"browser_provider": "camofox",
|
||||
"post_setup": "camofox",
|
||||
},
|
||||
{
|
||||
"name": "Browser Use",
|
||||
"badge": "free · local · cloud",
|
||||
"tag": "New SOTA web harness (CLI 3.0)",
|
||||
"env_vars": [],
|
||||
"browser_backend": "browser-use",
|
||||
"post_setup": "browser_use_cli",
|
||||
},
|
||||
],
|
||||
},
|
||||
"homeassistant": {
|
||||
|
|
@ -1760,6 +1769,18 @@ def _run_post_setup(post_setup_key: str):
|
|||
_print_warning(f" Chromium install failed: {exc}")
|
||||
_print_info(" Run manually: npx agent-browser install --with-deps")
|
||||
|
||||
elif post_setup_key == "browser_use_cli":
|
||||
if shutil.which("browser-use"):
|
||||
_print_success(" browser-use CLI found on PATH")
|
||||
elif shutil.which("uvx"):
|
||||
_print_info(" browser-use CLI not installed — it will run via `uvx browser-use`")
|
||||
_print_info(" For a persistent install: uv tool install browser-use")
|
||||
else:
|
||||
_print_warning(" browser-use CLI not found and uvx is unavailable")
|
||||
_print_info(" Install with: uv tool install browser-use (https://docs.astral.sh/uv/)")
|
||||
_print_info(" Local Chrome needs remote debugging: chrome://inspect/#remote-debugging")
|
||||
_print_info(" Cloud browsers: browser-use auth login (or set BROWSER_USE_API_KEY)")
|
||||
|
||||
elif post_setup_key == "camofox":
|
||||
camofox_dir = PROJECT_ROOT / "node_modules" / "@askjo" / "camofox-browser"
|
||||
_npm_bin = find_node_executable("npm")
|
||||
|
|
@ -3623,6 +3644,8 @@ def _is_provider_active(
|
|||
and cfg_get(config, "stt", "provider") == provider["stt_provider"]
|
||||
)
|
||||
if "browser_provider" in provider:
|
||||
if cfg_get(config, "browser", "backend"):
|
||||
return False
|
||||
current = cfg_get(config, "browser", "cloud_provider")
|
||||
return feature.managed_by_nous and provider["browser_provider"] == current
|
||||
if provider.get("web_backend"):
|
||||
|
|
@ -3637,8 +3660,24 @@ def _is_provider_active(
|
|||
current = cfg_get(config, "stt", "provider") or "local"
|
||||
return current == provider["stt_provider"]
|
||||
if "browser_provider" in provider:
|
||||
if cfg_get(config, "browser", "backend"):
|
||||
return False
|
||||
current = cfg_get(config, "browser", "cloud_provider")
|
||||
return provider["browser_provider"] == current
|
||||
if provider.get("browser_backend"):
|
||||
if cfg_get(config, "browser", "backend") == provider["browser_backend"]:
|
||||
return True
|
||||
# Legacy direct-API Browser Use cloud auto-routes to CLI
|
||||
try:
|
||||
from tools.browser_use_cli import is_legacy_browser_use_cloud_config
|
||||
|
||||
browser_cfg = config.get("browser") if isinstance(config, dict) else None
|
||||
return (
|
||||
provider["browser_backend"] == "browser-use"
|
||||
and is_legacy_browser_use_cloud_config(browser_cfg or {})
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
if provider.get("web_backend"):
|
||||
current = cfg_get(config, "web", "backend")
|
||||
return current == provider["web_backend"]
|
||||
|
|
@ -4084,8 +4123,15 @@ def _write_provider_config(provider: dict, config: dict, *, managed_feature) ->
|
|||
browser_cfg = config.setdefault("browser", {})
|
||||
if bp:
|
||||
browser_cfg["cloud_provider"] = bp
|
||||
# Leaving Browser Use CLI mode
|
||||
browser_cfg.pop("backend", None)
|
||||
browser_cfg["use_gateway"] = bool(managed_feature)
|
||||
|
||||
if provider.get("browser_backend"):
|
||||
browser_cfg = config.setdefault("browser", {})
|
||||
browser_cfg["backend"] = provider["browser_backend"]
|
||||
browser_cfg["use_gateway"] = False
|
||||
|
||||
# Set web search backend in config if applicable
|
||||
if provider.get("web_backend"):
|
||||
web_cfg = config.setdefault("web", {})
|
||||
|
|
@ -4229,6 +4275,9 @@ def _configure_provider(
|
|||
elif bp:
|
||||
_print_success(f" Browser cloud provider set to: {bp}")
|
||||
|
||||
if provider.get("browser_backend"):
|
||||
_print_success(" Browser set to Browser Use (browser_exec via CLI 3.0)")
|
||||
|
||||
# Set web search backend in config if applicable
|
||||
if provider.get("web_backend"):
|
||||
_print_success(f" Web backend set to: {provider['web_backend']}")
|
||||
|
|
@ -4741,8 +4790,16 @@ def _reconfigure_provider(
|
|||
elif bp:
|
||||
browser_cfg["cloud_provider"] = bp
|
||||
_print_success(f" Browser cloud provider set to: {bp}")
|
||||
# Leaving Browser Use CLI mode
|
||||
browser_cfg.pop("backend", None)
|
||||
browser_cfg["use_gateway"] = bool(managed_feature)
|
||||
|
||||
if provider.get("browser_backend"):
|
||||
browser_cfg = config.setdefault("browser", {})
|
||||
browser_cfg["backend"] = provider["browser_backend"]
|
||||
browser_cfg["use_gateway"] = False
|
||||
_print_success(" Browser set to Browser Use (browser_exec via CLI 3.0)")
|
||||
|
||||
# Set web search backend in config if applicable
|
||||
if provider.get("web_backend"):
|
||||
web_cfg = config.setdefault("web", {})
|
||||
|
|
|
|||
|
|
@ -306,19 +306,9 @@ class BrowserUseBrowserProvider(BrowserProvider):
|
|||
"Emergency cleanup failed for Browser Use session %s: %s", session_id, e
|
||||
)
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "Browser Use",
|
||||
"badge": "paid",
|
||||
"tag": "Cloud browser with remote execution",
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "BROWSER_USE_API_KEY",
|
||||
"prompt": "Browser Use API key",
|
||||
"url": "https://browser-use.com",
|
||||
},
|
||||
],
|
||||
# Cloud-scoped hook: installs the agent-browser CLI only (no
|
||||
# local Chromium — Browser Use hosts the browser).
|
||||
"post_setup": "browserbase",
|
||||
}
|
||||
def get_setup_schema(self) -> Optional[Dict[str, Any]]:
|
||||
# Hidden from the hermes tools picker: the "Browser Use" row now
|
||||
# activates the CLI-based backend (tools/browser_use_cli.py). This
|
||||
# provider stays registered for the Nous gateway path and un-migrated
|
||||
# legacy cloud_provider configs.
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -100,6 +100,54 @@ class TestBundledPluginsRegister:
|
|||
assert provider.name == plugin_name
|
||||
assert provider.display_name == expected_display
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["browserbase", "firecrawl"],
|
||||
)
|
||||
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
|
||||
"""``get_setup_schema()`` returns a dict the picker can consume."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
schema = provider.get_setup_schema()
|
||||
assert isinstance(schema, dict)
|
||||
assert "name" in schema
|
||||
assert "env_vars" in schema
|
||||
# Every cloud-browser plugin carries a post-setup hook so the
|
||||
# picker can auto-install its CLI dependency on selection.
|
||||
assert schema.get("post_setup")
|
||||
|
||||
def test_browser_use_hidden_from_picker(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
provider = get_provider("browser-use")
|
||||
assert provider is not None
|
||||
assert provider.get_setup_schema() is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["browserbase", "browser-use", "firecrawl"],
|
||||
)
|
||||
def test_each_plugin_implements_full_lifecycle(self, plugin_name: str) -> None:
|
||||
"""The ABC's three lifecycle methods are all overridden."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_provider import BrowserProvider
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
# Each method must be a real override, not the ABC's NotImplementedError
|
||||
# default — we check by comparing the function reference.
|
||||
assert type(provider).create_session is not BrowserProvider.create_session
|
||||
assert type(provider).close_session is not BrowserProvider.close_session
|
||||
assert (
|
||||
type(provider).emergency_cleanup is not BrowserProvider.emergency_cleanup
|
||||
)
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_available() behavior
|
||||
|
|
@ -244,6 +292,6 @@ class TestPickerIntegration:
|
|||
|
||||
rows = _plugin_browser_providers()
|
||||
names = sorted(r.get("browser_provider") for r in rows)
|
||||
assert names == ["browser-use", "browserbase", "firecrawl"]
|
||||
assert names == ["browserbase", "firecrawl"]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,498 @@
|
|||
"""Tests for the Browser Use CLI 3.0 backend (tools/browser_use_cli.py).
|
||||
|
||||
Covers the three seams the integration relies on:
|
||||
|
||||
* Mode detection — ``browser.backend: browser-use`` in config (set via the
|
||||
``hermes tools`` picker); off by default.
|
||||
* Tool-surface swap — when the mode is on, ``check_browser_requirements``
|
||||
returns False so every legacy ``browser_*`` tool (including
|
||||
browser_cdp/browser_dialog, whose check_fns funnel through it) is hidden,
|
||||
and ``browser_exec`` is advertised instead.
|
||||
* ``browser_exec`` execution — code is piped on stdin, ``session`` becomes
|
||||
``BU_NAME``, bad session names and a missing CLI produce actionable errors.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.browser_use_cli as bu_cli
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch):
|
||||
monkeypatch.delenv("BU_NAME", raising=False)
|
||||
monkeypatch.delenv("BU_AUTOSPAWN", raising=False)
|
||||
monkeypatch.delenv("BROWSER_USE_API_KEY", raising=False)
|
||||
yield
|
||||
|
||||
|
||||
def _fake_cli(tmp_path, body):
|
||||
"""Write an executable fake browser-use CLI and return its path."""
|
||||
script = tmp_path / "browser-use"
|
||||
script.write_text("#!/bin/sh\n" + body)
|
||||
script.chmod(script.stat().st_mode | stat.S_IXUSR)
|
||||
return str(script)
|
||||
|
||||
|
||||
class TestModeDetection:
|
||||
def test_off_by_default(self, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {})
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_config_opt_in(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"backend": "browser-use"}},
|
||||
)
|
||||
assert bu_cli.is_browser_use_cli_mode() is True
|
||||
|
||||
def test_other_backend_value_is_not_cli_mode(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"backend": "something-else"}},
|
||||
)
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_config_read_failure_fails_safe(self, monkeypatch):
|
||||
def boom():
|
||||
raise RuntimeError("config unreadable")
|
||||
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", boom)
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
|
||||
class TestToolSurfaceSwap:
|
||||
def test_legacy_browser_tools_hidden_in_cli_mode(self, monkeypatch):
|
||||
import tools.browser_tool as browser_tool
|
||||
|
||||
monkeypatch.setattr(browser_tool, "_is_browser_use_cli_mode", lambda: True)
|
||||
assert browser_tool.check_browser_requirements() is False
|
||||
assert browser_tool.check_browser_vision_requirements() is False
|
||||
|
||||
def test_browser_exec_registered_with_mode_check(self):
|
||||
from tools.registry import registry
|
||||
|
||||
entry = registry.get_entry("browser_exec")
|
||||
assert entry is not None
|
||||
assert entry.check_fn is bu_cli.is_browser_use_cli_mode
|
||||
assert entry.toolset == "browser-use"
|
||||
|
||||
def test_browser_exec_in_browser_toolsets(self):
|
||||
from toolsets import TOOLSETS, _HERMES_CORE_TOOLS
|
||||
|
||||
assert "browser_exec" in _HERMES_CORE_TOOLS
|
||||
assert "browser_exec" in TOOLSETS["browser"]["tools"]
|
||||
assert "browser_exec" in TOOLSETS["coding"]["tools"]
|
||||
|
||||
|
||||
class TestFindCli:
|
||||
def test_prefers_installed_binary(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
bu_cli.shutil, "which",
|
||||
lambda name: "/usr/local/bin/browser-use" if name == "browser-use" else "/usr/local/bin/uvx",
|
||||
)
|
||||
assert bu_cli._find_cli() == ["/usr/local/bin/browser-use"]
|
||||
|
||||
def test_falls_back_to_uvx(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
bu_cli.shutil, "which",
|
||||
lambda name: "/usr/local/bin/uvx" if name == "uvx" else None,
|
||||
)
|
||||
assert bu_cli._find_cli() == ["/usr/local/bin/uvx", "browser-use"]
|
||||
|
||||
def test_none_when_neither_available(self, monkeypatch):
|
||||
monkeypatch.setattr(bu_cli.shutil, "which", lambda name: None)
|
||||
assert bu_cli._find_cli() is None
|
||||
|
||||
|
||||
class TestLegacyCloudMigration:
|
||||
"""Pre-CLI direct-API Browser Use cloud configs (cloud_provider:
|
||||
"browser-use" + BROWSER_USE_API_KEY) auto-route to the CLI backend;
|
||||
Nous-gateway users stay on the legacy provider path."""
|
||||
|
||||
_LEGACY = {"browser": {"cloud_provider": "browser-use"}}
|
||||
|
||||
def test_direct_api_config_migrates(self, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: self._LEGACY)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
assert bu_cli.is_browser_use_cli_mode() is True
|
||||
|
||||
def test_gateway_config_stays_on_legacy_path(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"cloud_provider": "browser-use", "use_gateway": True}},
|
||||
)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_no_api_key_stays_on_legacy_path(self, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: self._LEGACY)
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_explicit_other_backend_wins(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"cloud_provider": "browser-use", "backend": "something-else"}},
|
||||
)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_other_cloud_provider_does_not_migrate(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"cloud_provider": "browserbase"}},
|
||||
)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_explicit_local_does_not_migrate(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"cloud_provider": "local"}},
|
||||
)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_auto_detect_with_key_migrates(self, monkeypatch):
|
||||
"""No cloud_provider configured + BROWSER_USE_API_KEY set: credential
|
||||
auto-detection prefers Browser Use (even when Browserbase creds are
|
||||
also present), which now means Browser Use mode."""
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {})
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
monkeypatch.setenv("BROWSERBASE_API_KEY", "bb-key")
|
||||
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "bb-project")
|
||||
assert bu_cli.is_browser_use_cli_mode() is True
|
||||
|
||||
def test_auto_detect_without_key_does_not_migrate(self, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {})
|
||||
assert bu_cli.is_browser_use_cli_mode() is False
|
||||
|
||||
def test_migrated_config_gets_bu_autospawn(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: self._LEGACY)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
cli = _fake_cli(tmp_path, 'cat > /dev/null\necho "autospawn:$BU_AUTOSPAWN"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec("print(1)"))
|
||||
assert "autospawn:1" in result["output"]
|
||||
|
||||
def test_explicit_backend_does_not_set_bu_autospawn(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.read_raw_config",
|
||||
lambda: {"browser": {"backend": "browser-use"}},
|
||||
)
|
||||
cli = _fake_cli(tmp_path, 'cat > /dev/null\necho "autospawn:[$BU_AUTOSPAWN]"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec("print(1)"))
|
||||
assert "autospawn:[]" in result["output"]
|
||||
|
||||
def test_picker_highlights_cli_row_for_migrated_config(self, monkeypatch):
|
||||
from hermes_cli.tools_config import TOOL_CATEGORIES, _is_provider_active
|
||||
|
||||
cli_row = next(
|
||||
r for r in TOOL_CATEGORIES["browser"]["providers"] if r.get("browser_backend")
|
||||
)
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "bu-key")
|
||||
assert _is_provider_active(cli_row, dict(self._LEGACY)) is True
|
||||
monkeypatch.delenv("BROWSER_USE_API_KEY")
|
||||
assert _is_provider_active(cli_row, dict(self._LEGACY)) is False
|
||||
|
||||
|
||||
class TestProviderPickerIntegration:
|
||||
"""The `hermes tools` Browser Automation picker row (browser_backend
|
||||
marker) must enter/leave CLI mode cleanly and highlight correctly."""
|
||||
|
||||
def _rows(self):
|
||||
from hermes_cli.tools_config import TOOL_CATEGORIES
|
||||
|
||||
return TOOL_CATEGORIES["browser"]["providers"]
|
||||
|
||||
def test_picker_has_browser_use_cli_row(self):
|
||||
row = next(r for r in self._rows() if r.get("browser_backend"))
|
||||
assert row["browser_backend"] == "browser-use"
|
||||
assert row["name"] == "Browser Use"
|
||||
|
||||
def test_picker_row_names_stay_unique(self):
|
||||
"""The CLI row is named "Browser Use"; the legacy plugin API row must
|
||||
keep a distinct name — apply_provider_selection matches by name."""
|
||||
from hermes_cli.tools_config import TOOL_CATEGORIES, _plugin_browser_providers
|
||||
|
||||
names = [r["name"] for r in TOOL_CATEGORIES["browser"]["providers"]]
|
||||
names += [r["name"] for r in _plugin_browser_providers()]
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
def test_selecting_cli_row_writes_backend_and_keeps_cloud_provider(self):
|
||||
from hermes_cli.tools_config import _write_provider_config
|
||||
|
||||
row = next(r for r in self._rows() if r.get("browser_backend"))
|
||||
config = {"browser": {"cloud_provider": "browserbase"}}
|
||||
assert row["name"] == "Browser Use"
|
||||
_write_provider_config(row, config, managed_feature=None)
|
||||
assert config["browser"]["backend"] == "browser-use"
|
||||
assert config["browser"]["cloud_provider"] == "browserbase"
|
||||
|
||||
def test_selecting_provider_row_leaves_cli_mode(self):
|
||||
from hermes_cli.tools_config import _write_provider_config
|
||||
|
||||
local_row = next(
|
||||
r for r in self._rows() if r.get("browser_provider") == "local"
|
||||
)
|
||||
config = {"browser": {"backend": "browser-use"}}
|
||||
_write_provider_config(local_row, config, managed_feature=None)
|
||||
assert "backend" not in config["browser"]
|
||||
assert config["browser"]["cloud_provider"] == "local"
|
||||
|
||||
def test_active_row_highlight_is_mutually_exclusive(self):
|
||||
from hermes_cli.tools_config import _is_provider_active
|
||||
|
||||
cli_row = next(r for r in self._rows() if r.get("browser_backend"))
|
||||
local_row = next(
|
||||
r for r in self._rows() if r.get("browser_provider") == "local"
|
||||
)
|
||||
cli_config = {"browser": {"cloud_provider": "local", "backend": "browser-use"}}
|
||||
assert _is_provider_active(cli_row, cli_config) is True
|
||||
assert _is_provider_active(local_row, cli_config) is False
|
||||
|
||||
local_config = {"browser": {"cloud_provider": "local"}}
|
||||
assert _is_provider_active(cli_row, local_config) is False
|
||||
assert _is_provider_active(local_row, local_config) is True
|
||||
|
||||
|
||||
class TestBrowserUseSlashCommand:
|
||||
"""/browser use [off] toggles browser.backend and resets the session,
|
||||
mirroring the /tools enable/disable flow."""
|
||||
|
||||
class _Stub:
|
||||
def __init__(self):
|
||||
self.session_resets = 0
|
||||
|
||||
def new_session(self):
|
||||
self.session_resets += 1
|
||||
|
||||
def _run(self, cmd, config, monkeypatch):
|
||||
import hermes_cli.config as hc
|
||||
from hermes_cli.cli_commands_mixin import CLICommandsMixin
|
||||
|
||||
saved = {}
|
||||
monkeypatch.setattr(hc, "load_config", lambda: config)
|
||||
monkeypatch.setattr(hc, "save_config", lambda c: saved.update(c))
|
||||
stub = self._Stub()
|
||||
CLICommandsMixin._handle_browser_command(stub, cmd)
|
||||
return stub, saved
|
||||
|
||||
def test_use_enables_backend_and_resets_session(self, monkeypatch):
|
||||
stub, saved = self._run("/browser use", {}, monkeypatch)
|
||||
assert saved["browser"]["backend"] == "browser-use"
|
||||
assert stub.session_resets == 1
|
||||
|
||||
def test_use_off_removes_backend(self, monkeypatch):
|
||||
config = {"browser": {"backend": "browser-use"}}
|
||||
stub, saved = self._run("/browser use off", config, monkeypatch)
|
||||
assert "backend" not in saved["browser"]
|
||||
assert stub.session_resets == 1
|
||||
|
||||
def test_use_bad_arg_prints_usage_without_writing(self, monkeypatch):
|
||||
stub, saved = self._run("/browser use whatever", {}, monkeypatch)
|
||||
assert saved == {}
|
||||
assert stub.session_resets == 0
|
||||
|
||||
|
||||
class TestNativeScreenshots:
|
||||
"""Screenshots printed by capture_screenshot() attach directly to the
|
||||
model's context when it has native vision — no aux vision-LLM detour."""
|
||||
|
||||
def _shot(self, tmp_path):
|
||||
shot = tmp_path / "shot.png"
|
||||
shot.write_bytes(b"\x89PNG fake")
|
||||
return str(shot)
|
||||
|
||||
def test_find_screenshot_returns_last_fresh_path(self, tmp_path):
|
||||
a, b = self._shot(tmp_path), str(tmp_path / "b.png")
|
||||
(tmp_path / "b.png").write_bytes(b"\x89PNG fake2")
|
||||
out = f"step one saved {a}\nthen saved {b}\n"
|
||||
assert bu_cli._find_screenshot(out, since=time.time() - 5) == b
|
||||
|
||||
def test_find_screenshot_rejects_stale_and_missing(self, tmp_path):
|
||||
stale = self._shot(tmp_path)
|
||||
os.utime(stale, (time.time() - 900, time.time() - 900))
|
||||
out = f"{stale}\n/nonexistent/dir/x.png\n"
|
||||
assert bu_cli._find_screenshot(out, since=time.time()) is None
|
||||
|
||||
def test_vision_model_gets_multimodal_envelope(self, tmp_path, monkeypatch):
|
||||
shot = self._shot(tmp_path)
|
||||
cli = _fake_cli(tmp_path, f'cat > /dev/null\necho "{shot}"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
monkeypatch.setattr(
|
||||
"tools.vision_tools._should_use_native_vision_fast_path", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"tools.vision_tools._resize_image_for_vision",
|
||||
lambda p, **kw: "data:image/png;base64,QUJD",
|
||||
)
|
||||
result = bu_cli.browser_exec("print(capture_screenshot())")
|
||||
assert isinstance(result, dict) and result["_multimodal"] is True
|
||||
kinds = [part["type"] for part in result["content"]]
|
||||
assert kinds == ["text", "image_url"]
|
||||
assert result["meta"]["screenshot_path"] == shot
|
||||
assert shot in result["text_summary"]
|
||||
|
||||
def test_text_only_model_gets_plain_result_with_path(self, tmp_path, monkeypatch):
|
||||
shot = self._shot(tmp_path)
|
||||
cli = _fake_cli(tmp_path, f'cat > /dev/null\necho "{shot}"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
monkeypatch.setattr(
|
||||
"tools.vision_tools._should_use_native_vision_fast_path", lambda: False
|
||||
)
|
||||
result = json.loads(bu_cli.browser_exec("print(capture_screenshot())"))
|
||||
assert result["screenshot_path"] == shot
|
||||
|
||||
def test_no_screenshot_keeps_string_result(self, tmp_path, monkeypatch):
|
||||
cli = _fake_cli(tmp_path, 'cat > /dev/null\necho "no images here"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec("print(1)"))
|
||||
assert "screenshot_path" not in result
|
||||
|
||||
|
||||
class TestStepLabels:
|
||||
"""browser_exec code leads with a `# …` comment (per the tool
|
||||
description); the TUI surfaces it as the step label and keeps the code
|
||||
collapsed behind display.tool_preview_length."""
|
||||
|
||||
_CODE = "# Searching Amazon for paper towels\nnew_tab('https://amazon.com')\nwait_for_load()"
|
||||
|
||||
def test_leading_comment_becomes_step_label(self):
|
||||
from agent.display import _browser_exec_step_label
|
||||
|
||||
assert _browser_exec_step_label({"code": self._CODE}) == "Searching Amazon for paper towels"
|
||||
|
||||
def test_no_comment_returns_none(self):
|
||||
from agent.display import _browser_exec_step_label
|
||||
|
||||
assert _browser_exec_step_label({"code": "new_tab('x')"}) is None
|
||||
assert _browser_exec_step_label({"code": ""}) is None
|
||||
assert _browser_exec_step_label({"code": "# "}) is None
|
||||
|
||||
def test_label_hard_capped_regardless_of_global_setting(self):
|
||||
from agent.display import _browser_exec_step_label
|
||||
|
||||
long = "# " + "x" * 200
|
||||
label = _browser_exec_step_label({"code": long})
|
||||
assert len(label) <= 80 and label.endswith("…")
|
||||
|
||||
def test_preview_prefers_comment_over_code(self):
|
||||
from agent.display import build_tool_preview
|
||||
|
||||
assert build_tool_preview("browser_exec", {"code": self._CODE}) == (
|
||||
"Searching Amazon for paper towels"
|
||||
)
|
||||
assert "new_tab" in build_tool_preview("browser_exec", {"code": "new_tab('x')"})
|
||||
|
||||
def test_progress_line_shows_label(self):
|
||||
from agent.display import get_cute_tool_message
|
||||
|
||||
line = get_cute_tool_message("browser_exec", {"code": self._CODE}, 1.2)
|
||||
assert "Searching Amazon for paper towels" in line
|
||||
assert "new_tab" not in line
|
||||
|
||||
def test_header_instructs_leading_comment(self):
|
||||
assert "one-line comment" in bu_cli._HEADER_BASE
|
||||
assert "step label" in bu_cli._HEADER_BASE
|
||||
|
||||
|
||||
class TestHeaderVariants:
|
||||
def test_vision_header_forbids_vision_tool_detour(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"tools.vision_tools._should_use_native_vision_fast_path", lambda: True
|
||||
)
|
||||
header = bu_cli._description_header()
|
||||
assert header.startswith(bu_cli._HEADER_BASE)
|
||||
assert "attached to your context automatically" in header
|
||||
|
||||
def test_text_only_header_teaches_text_workflow(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"tools.vision_tools._should_use_native_vision_fast_path", lambda: False
|
||||
)
|
||||
header = bu_cli._description_header()
|
||||
assert "cannot view images" in header
|
||||
assert "page_info()" in header
|
||||
|
||||
|
||||
class TestSkillTextDescription:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_skill_cache(self, monkeypatch):
|
||||
monkeypatch.setattr(bu_cli, "_skill_text_cache", None)
|
||||
monkeypatch.setattr(bu_cli, "_skill_text_fetched", False)
|
||||
yield
|
||||
|
||||
def test_description_is_verbatim_cli_skill_text(self, tmp_path, monkeypatch):
|
||||
cli = _fake_cli(
|
||||
tmp_path,
|
||||
'if [ "$1" = "skill" ]; then echo "# Browser Use\nverbatim skill body"; fi\n',
|
||||
)
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
overrides = bu_cli._dynamic_schema_overrides()
|
||||
assert overrides["description"].startswith(bu_cli._DESCRIPTION_HEADER)
|
||||
assert overrides["description"].endswith("# Browser Use\nverbatim skill body")
|
||||
|
||||
def test_skill_text_cached_after_first_fetch(self, tmp_path, monkeypatch):
|
||||
calls = []
|
||||
cli = _fake_cli(tmp_path, 'echo "skill text"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: (calls.append(1), [cli])[1])
|
||||
assert "skill text" in bu_cli._cli_skill_text()
|
||||
assert "skill text" in bu_cli._cli_skill_text()
|
||||
assert len(calls) == 1
|
||||
|
||||
def test_no_override_when_cli_missing(self, monkeypatch):
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: None)
|
||||
assert bu_cli._dynamic_schema_overrides() == {}
|
||||
|
||||
|
||||
class TestBrowserExec:
|
||||
def test_missing_cli_returns_install_hint(self, monkeypatch):
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: None)
|
||||
result = json.loads(bu_cli.browser_exec("print(page_info())"))
|
||||
assert "uv tool install browser-use" in result["error"]
|
||||
|
||||
def test_empty_code_rejected(self):
|
||||
result = json.loads(bu_cli.browser_exec(" "))
|
||||
assert "error" in result
|
||||
|
||||
def test_code_piped_on_stdin(self, tmp_path, monkeypatch):
|
||||
cli = _fake_cli(tmp_path, 'code=$(cat)\necho "got:$code"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec('print("hi")'))
|
||||
assert result["success"] is True
|
||||
assert result["exit_code"] == 0
|
||||
assert 'got:print("hi")' in result["output"]
|
||||
assert "session" not in result
|
||||
|
||||
def test_session_sets_bu_name(self, tmp_path, monkeypatch):
|
||||
cli = _fake_cli(tmp_path, 'cat > /dev/null\necho "bu:$BU_NAME"\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec("print(1)", session="r7k2"))
|
||||
assert "bu:r7k2" in result["output"]
|
||||
assert result["session"] == "r7k2"
|
||||
|
||||
def test_invalid_session_name_rejected(self, monkeypatch, tmp_path):
|
||||
cli = _fake_cli(tmp_path, "cat > /dev/null\n")
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec("print(1)", session="bad name!"))
|
||||
assert "error" in result
|
||||
assert "session" in result["error"].lower()
|
||||
|
||||
def test_nonzero_exit_reports_failure_and_stderr(self, tmp_path, monkeypatch):
|
||||
cli = _fake_cli(tmp_path, 'cat > /dev/null\necho "boom" >&2\nexit 3\n')
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
result = json.loads(bu_cli.browser_exec("print(1)"))
|
||||
assert result["success"] is False
|
||||
assert result["exit_code"] == 3
|
||||
assert "boom" in result["stderr"]
|
||||
|
||||
def test_timeout_returns_actionable_error(self, tmp_path, monkeypatch):
|
||||
cli = _fake_cli(tmp_path, "cat > /dev/null\nsleep 30\n")
|
||||
monkeypatch.setattr(bu_cli, "_find_cli", lambda: [cli])
|
||||
monkeypatch.setattr(bu_cli, "_MIN_TIMEOUT_S", 1)
|
||||
result = json.loads(bu_cli.browser_exec("print(1)", timeout_s=1))
|
||||
assert "timed out" in result["error"]
|
||||
|
|
@ -182,6 +182,11 @@ try:
|
|||
from tools.browser_camofox import is_camofox_mode as _is_camofox_mode
|
||||
except ImportError:
|
||||
_is_camofox_mode = lambda: False # noqa: E731
|
||||
# Browser Use CLI (optional)
|
||||
try:
|
||||
from tools.browser_use_cli import is_browser_use_cli_mode as _is_browser_use_cli_mode
|
||||
except ImportError:
|
||||
_is_browser_use_cli_mode = lambda: False # noqa: E731
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -4885,6 +4890,12 @@ def check_browser_requirements() -> bool:
|
|||
Returns:
|
||||
True if all requirements are met, False otherwise
|
||||
"""
|
||||
# Browser Use CLI backend — browser_exec replaces the whole browser_*
|
||||
# surface (including browser_cdp/browser_dialog, whose check_fns funnel
|
||||
# through here), so hide these tools from the model.
|
||||
if _is_browser_use_cli_mode():
|
||||
return False
|
||||
|
||||
# Camofox backend — only needs the server URL, no agent-browser CLI
|
||||
if _is_camofox_mode():
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,337 @@
|
|||
"""Use the Browser Use CLI 3.0 (https://browser-use.com) for browser automation
|
||||
|
||||
When browser.backend is "browser-use", the model gets ``browser_exec`` tool
|
||||
instead of default browser tools
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from utils import is_truthy_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BACKEND_KEY = "browser-use"
|
||||
|
||||
# Cloud daemon names become the BU_NAME env var
|
||||
_SESSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
|
||||
|
||||
_DEFAULT_TIMEOUT_S = 120
|
||||
_MIN_TIMEOUT_S = 5
|
||||
_MAX_TIMEOUT_S = 600
|
||||
_STDERR_CAP_CHARS = 4000
|
||||
|
||||
# Screenshot paths printed by capture_screenshot() in the exec output
|
||||
_IMAGE_PATH_RE = re.compile(r"(/[^\s\"']+?\.(?:png|jpe?g|webp))", re.IGNORECASE)
|
||||
|
||||
|
||||
def _read_browser_cfg() -> dict:
|
||||
"""Return the ``browser:`` config section, or {} on any failure."""
|
||||
try:
|
||||
from hermes_cli.config import cfg_get, read_raw_config
|
||||
|
||||
cfg = cfg_get(read_raw_config(), "browser", default={})
|
||||
return cfg if isinstance(cfg, dict) else {}
|
||||
except Exception as e:
|
||||
logger.debug("Could not read browser config section: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def get_browser_backend() -> str:
|
||||
"""Return the configured browser backend key ("" = legacy stack)."""
|
||||
return str(_read_browser_cfg().get("backend") or "").strip().lower()
|
||||
|
||||
|
||||
def is_legacy_browser_use_cloud_config(browser_cfg: dict) -> bool:
|
||||
"""True for pre-CLI direct-API Browser Use cloud configs"""
|
||||
if not isinstance(browser_cfg, dict):
|
||||
return False
|
||||
if browser_cfg.get("backend"):
|
||||
return False # an explicit backend choice wins
|
||||
provider = str(browser_cfg.get("cloud_provider") or "").strip().lower()
|
||||
if provider not in {"browser-use", ""}:
|
||||
return False # explicit local/Browserbase/… choices win
|
||||
if is_truthy_value(browser_cfg.get("use_gateway"), default=False):
|
||||
return False
|
||||
return bool(os.getenv("BROWSER_USE_API_KEY"))
|
||||
|
||||
|
||||
def is_browser_use_cli_mode() -> bool:
|
||||
"""True when the Browser Use CLI replaces the built-in browser stack"""
|
||||
backend = get_browser_backend()
|
||||
if backend:
|
||||
return backend == _BACKEND_KEY
|
||||
return is_legacy_browser_use_cloud_config(_read_browser_cfg())
|
||||
|
||||
|
||||
def _find_cli() -> Optional[List[str]]:
|
||||
"""Locate the browser-use CLI, or None when it can't be run.
|
||||
|
||||
Prefers an installed browser-use binary; falls back to running it
|
||||
through uvx
|
||||
"""
|
||||
direct = shutil.which("browser-use")
|
||||
if direct:
|
||||
return [direct]
|
||||
uvx = shutil.which("uvx")
|
||||
if uvx:
|
||||
return [uvx, "browser-use"]
|
||||
return None
|
||||
|
||||
|
||||
def _find_screenshot(stdout: str, since: float) -> Optional[str]:
|
||||
"""Return the last screenshot path printed during this exec, or None.
|
||||
|
||||
Only accepts files that exist and were written after the exec started
|
||||
"""
|
||||
for path in reversed(_IMAGE_PATH_RE.findall(stdout or "")):
|
||||
try:
|
||||
if os.path.isfile(path) and os.path.getmtime(path) >= since - 1:
|
||||
return path
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _native_screenshot_result(result: Dict[str, Any], path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Build a multimodal tool result attaching path for vision models"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
from tools.vision_tools import (
|
||||
_resize_image_for_vision,
|
||||
_should_use_native_vision_fast_path,
|
||||
)
|
||||
|
||||
if not _should_use_native_vision_fast_path():
|
||||
return None
|
||||
data_url = _resize_image_for_vision(Path(path))
|
||||
text = json.dumps(result, ensure_ascii=False)
|
||||
return {
|
||||
"_multimodal": True,
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
text
|
||||
+ "\n\nThe screenshot from this call is attached — "
|
||||
"inspect it with your native vision."
|
||||
),
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
],
|
||||
"text_summary": text,
|
||||
"meta": {"screenshot_path": path, "native_vision": True},
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug("Native screenshot attach failed (falling back to text): %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT_S):
|
||||
"""Run Python code through the browser-use CLI, and return its output"""
|
||||
from tools.registry import tool_error, tool_result
|
||||
|
||||
if not code or not code.strip():
|
||||
return tool_error("No code provided. Pass Python that uses the pre-imported helpers, e.g. new_tab(\"https://example.com\") then print(page_info()).")
|
||||
|
||||
cmd = _find_cli()
|
||||
if not cmd:
|
||||
return tool_error(
|
||||
"browser-use CLI not found on PATH, and uvx is unavailable for a "
|
||||
"zero-install run. Install it with `uv tool install browser-use` "
|
||||
"(or `pipx install browser-use`), then run `browser-use --doctor` "
|
||||
"to verify the setup."
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
if session:
|
||||
if not _SESSION_RE.match(session):
|
||||
return tool_error(
|
||||
f"Invalid session name {session!r}: use 1-64 letters, digits, "
|
||||
"dashes, or underscores (e.g. 'r7k2')."
|
||||
)
|
||||
env["BU_NAME"] = session
|
||||
|
||||
# BU_AUTOSPAWN makes the CLI start a Browser Use cloud browser when no
|
||||
# local Chrome/CDP endpoint is reachable (their API key authenticates it)
|
||||
if "BU_AUTOSPAWN" not in env and is_legacy_browser_use_cloud_config(_read_browser_cfg()):
|
||||
env["BU_AUTOSPAWN"] = "1"
|
||||
|
||||
try:
|
||||
timeout = max(_MIN_TIMEOUT_S, min(int(timeout_s), _MAX_TIMEOUT_S))
|
||||
except (TypeError, ValueError):
|
||||
timeout = _DEFAULT_TIMEOUT_S
|
||||
|
||||
started = time.time()
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
input=code,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env=env,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return tool_error(
|
||||
f"browser-use exec timed out after {timeout}s. The daemon may "
|
||||
"still be working; retry with a larger timeout_s, or split the "
|
||||
"code into smaller steps."
|
||||
)
|
||||
except OSError as e:
|
||||
return tool_error(f"Failed to launch browser-use CLI: {e}")
|
||||
|
||||
result = {
|
||||
"success": proc.returncode == 0,
|
||||
"exit_code": proc.returncode,
|
||||
"output": proc.stdout,
|
||||
}
|
||||
if session:
|
||||
result["session"] = session
|
||||
stderr = (proc.stderr or "").strip()
|
||||
if stderr:
|
||||
if len(stderr) > _STDERR_CAP_CHARS:
|
||||
stderr = stderr[:_STDERR_CAP_CHARS] + "\n… (stderr truncated)"
|
||||
result["stderr"] = stderr
|
||||
|
||||
screenshot = _find_screenshot(proc.stdout, started)
|
||||
if screenshot:
|
||||
result["screenshot_path"] = screenshot
|
||||
native = _native_screenshot_result(result, screenshot)
|
||||
if native is not None:
|
||||
return native
|
||||
return tool_result(result)
|
||||
|
||||
|
||||
# The tool description is the CLI's skill, fetched from browser-use skill
|
||||
_HEADER_BASE = (
|
||||
"Drive a real web browser via the Browser Use CLI. The `code` argument "
|
||||
"is piped verbatim to the `browser-use` CLI on stdin and executed with "
|
||||
"its pre-imported helpers; stdout comes back in the result. Start `code` "
|
||||
"with a one-line comment describing the step for the user in plain, "
|
||||
"non-technical language, max 60 chars (e.g. `# Searching Amazon for "
|
||||
"paper towels`) — the UI displays it as the step label. Batch whole "
|
||||
"sub-procedures (navigate, wait, extract, act) into ONE call — do not "
|
||||
"spend a call per action. js() takes a JS expression: js('document.title') "
|
||||
"or js('(() => {...})()') — a bare '() => {...}' returns the function "
|
||||
"itself, uncalled. The CLI's own documentation follows and is complete "
|
||||
"(no need to read separate browser-use skill files) — where it shows "
|
||||
"shell heredocs (browser-use <<'PY' … PY), pass the Python as `code` "
|
||||
"instead; where it shows BU_NAME=<name>, pass session=<name> instead."
|
||||
)
|
||||
|
||||
_HEADER_VISION = (
|
||||
" Screenshots are attached to your context automatically: when the exec "
|
||||
"output contains a capture_screenshot() path, the image arrives with "
|
||||
"this tool's result and you inspect it directly with your own vision — "
|
||||
"never send browser screenshots to a separate vision tool."
|
||||
)
|
||||
|
||||
_HEADER_TEXT_ONLY = (
|
||||
" Your model cannot view images, so work text-first: page_info() for "
|
||||
"state, js() for reading/extracting DOM text, fill_input(selector, "
|
||||
"text) for inputs, and js(\"document.querySelector('…').click()\") for "
|
||||
"clicks — skip the screenshot-driven workflow described below."
|
||||
)
|
||||
|
||||
_DESCRIPTION_HEADER = _HEADER_BASE
|
||||
|
||||
|
||||
def _description_header() -> str:
|
||||
"""Header tailored to whether the active model can see images natively"""
|
||||
try:
|
||||
from tools.vision_tools import _should_use_native_vision_fast_path
|
||||
|
||||
if _should_use_native_vision_fast_path():
|
||||
return _HEADER_BASE + _HEADER_VISION
|
||||
except Exception:
|
||||
pass
|
||||
return _HEADER_BASE + _HEADER_TEXT_ONLY
|
||||
|
||||
_skill_text_cache: Optional[str] = None
|
||||
_skill_text_fetched = False
|
||||
|
||||
|
||||
def _cli_skill_text() -> str:
|
||||
"""Return the installed CLI's skill"""
|
||||
global _skill_text_cache, _skill_text_fetched
|
||||
if _skill_text_fetched:
|
||||
return _skill_text_cache or ""
|
||||
_skill_text_fetched = True
|
||||
cmd = _find_cli()
|
||||
if not cmd:
|
||||
return ""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[*cmd, "skill"], capture_output=True, text=True, timeout=30
|
||||
)
|
||||
if proc.returncode == 0 and proc.stdout.strip():
|
||||
_skill_text_cache = proc.stdout.strip()
|
||||
except Exception as e:
|
||||
logger.debug("Could not fetch browser-use skill text: %s", e)
|
||||
return _skill_text_cache or ""
|
||||
|
||||
|
||||
def _dynamic_schema_overrides() -> dict:
|
||||
skill = _cli_skill_text()
|
||||
if not skill:
|
||||
return {}
|
||||
return {"description": _description_header() + "\n\n---\n\n" + skill}
|
||||
|
||||
|
||||
BROWSER_EXEC_SCHEMA = {
|
||||
"name": "browser_exec",
|
||||
# Static fallback, used only when the CLI (and uvx) is unavailable
|
||||
"description": (
|
||||
_HEADER_BASE
|
||||
+ "\n\n(The browser-use CLI is not installed yet, so its full skill "
|
||||
"documentation could not be loaded. Install it with "
|
||||
"`uv tool install browser-use`.)"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string",
|
||||
"description": "Python code to execute using the pre-imported browser helpers. Use print(...) for any data you need back.",
|
||||
},
|
||||
"session": {
|
||||
"type": "string",
|
||||
"description": "Named cloud browser session (sets BU_NAME). Omit for the local default daemon. Use the same name you passed to start_remote_daemon().",
|
||||
},
|
||||
"timeout_s": {
|
||||
"type": "integer",
|
||||
"description": f"Max seconds to wait for the code to finish (default {_DEFAULT_TIMEOUT_S}, max {_MAX_TIMEOUT_S}).",
|
||||
"default": _DEFAULT_TIMEOUT_S,
|
||||
},
|
||||
},
|
||||
"required": ["code"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
from tools.registry import registry
|
||||
|
||||
registry.register(
|
||||
name="browser_exec",
|
||||
toolset="browser-use",
|
||||
schema=BROWSER_EXEC_SCHEMA,
|
||||
handler=lambda args, **kw: browser_exec(
|
||||
code=args.get("code", ""),
|
||||
session=args.get("session", "") or "",
|
||||
timeout_s=args.get("timeout_s", _DEFAULT_TIMEOUT_S),
|
||||
),
|
||||
check_fn=is_browser_use_cli_mode,
|
||||
dynamic_schema_overrides=_dynamic_schema_overrides,
|
||||
emoji="🌐",
|
||||
)
|
||||
|
|
@ -55,6 +55,8 @@ _HERMES_CORE_TOOLS = [
|
|||
"browser_type", "browser_scroll", "browser_back",
|
||||
"browser_press", "browser_get_images",
|
||||
"browser_vision", "browser_console", "browser_cdp", "browser_dialog",
|
||||
# replaces other tools when browser.backend is "browser-use"
|
||||
"browser_exec",
|
||||
# Text-to-speech
|
||||
"text_to_speech",
|
||||
# Planning & memory
|
||||
|
|
@ -205,7 +207,7 @@ TOOLSETS = {
|
|||
"browser_type", "browser_scroll", "browser_back",
|
||||
"browser_press", "browser_get_images",
|
||||
"browser_vision", "browser_console", "browser_cdp",
|
||||
"browser_dialog", "web_search"
|
||||
"browser_dialog", "browser_exec", "web_search"
|
||||
],
|
||||
"includes": []
|
||||
},
|
||||
|
|
@ -410,6 +412,7 @@ TOOLSETS = {
|
|||
"browser_type", "browser_scroll", "browser_back",
|
||||
"browser_press", "browser_get_images",
|
||||
"browser_vision", "browser_console", "browser_cdp", "browser_dialog",
|
||||
"browser_exec",
|
||||
"todo", "memory",
|
||||
"session_search", "clarify",
|
||||
"execute_code", "delegate_task",
|
||||
|
|
@ -442,6 +445,7 @@ TOOLSETS = {
|
|||
"browser_type", "browser_scroll", "browser_back",
|
||||
"browser_press", "browser_get_images",
|
||||
"browser_vision", "browser_console", "browser_cdp", "browser_dialog",
|
||||
"browser_exec",
|
||||
"todo", "memory",
|
||||
"session_search",
|
||||
"execute_code", "delegate_task",
|
||||
|
|
@ -471,6 +475,7 @@ TOOLSETS = {
|
|||
"browser_type", "browser_scroll", "browser_back",
|
||||
"browser_press", "browser_get_images",
|
||||
"browser_vision", "browser_console", "browser_cdp", "browser_dialog",
|
||||
"browser_exec",
|
||||
# Planning & memory
|
||||
"todo", "memory",
|
||||
# Session history search
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ Hermes Agent includes a full browser automation toolset with multiple backend op
|
|||
|
||||
- **Browserbase cloud mode** via [Browserbase](https://browserbase.com) for managed cloud browsers and anti-bot tooling
|
||||
- **Browser Use cloud mode** via [Browser Use](https://browser-use.com) as an alternative cloud browser provider
|
||||
- **Browser Use mode** via the [Browser Use CLI 3.0](https://github.com/browser-use/browser-use) — a new browser harness that is SOTA for web tasks; automates your local Chrome or Browser Use cloud browsers
|
||||
- **Firecrawl cloud mode** via [Firecrawl](https://firecrawl.dev) for cloud browsers with built-in scraping
|
||||
- **Camofox local mode** via [Camofox](https://github.com/jo-inc/camofox-browser) for local anti-detection browsing (Firefox-based fingerprint spoofing)
|
||||
- **Local Chromium-family CDP** — connect browser tools to your own Chrome, Brave, Chromium, or Edge instance using `/browser connect`
|
||||
|
|
@ -58,7 +59,21 @@ To use Browser Use as your cloud browser provider, add:
|
|||
BROWSER_USE_API_KEY=***
|
||||
```
|
||||
|
||||
Get your API key at [browser-use.com](https://browser-use.com). Browser Use provides a cloud browser via its REST API. If both Browserbase and Browser Use credentials are set, Browserbase takes priority.
|
||||
Get your API key at [browser-use.com](https://browser-use.com).
|
||||
|
||||
### Browser Use mode
|
||||
|
||||
Browser Use mode uses the [Browser Use CLI 3.0](https://github.com/browser-use/browser-use) — a new browser harness that is state-of-the-art at web tasks — instead of the default browser tools. The agent writes and executes Python in the browser to click, type, drag, scrape, and interact with webpages. It works in your local browser or with Browser Use cloud browsers.
|
||||
|
||||
Enable it with `hermes tools` → **Browser Automation → Browser Use** (free · local · cloud), or directly:
|
||||
|
||||
```yaml
|
||||
# Add to ~/.hermes/config.yaml
|
||||
browser:
|
||||
backend: "browser-use"
|
||||
```
|
||||
|
||||
Cloud browsers need `browser-use auth login` or `BROWSER_USE_API_KEY`.
|
||||
|
||||
### Firecrawl cloud mode
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue