```
feat(agent): 添加技能自动路由功能 - 引入 SkillRouter 实现根据用户消息自动推荐技能 - 在构建引擎时集成技能路由器 - 从对话历史中提取最后一条用户消息作为路由输入 - 为技能添加触发关键词和中文标题字段支持 refactor(browser): 重构浏览器自动化为子进程架构 - 将 Playwright 浏览器控制移至独立的子进程 worker - 解决 PyInstaller 打包环境下 C 扩展兼容性问题 - 通过 JSON RPC 协议与浏览器 worker 通信 - 添加工具目录和脚本路径查找机制 feat(skills): 增强技能元数据和UI展示 - 为技能添加 triggers 和 title 字段 - 在技能商店中包含中文标题信息 - 添加 office-viz 技能优先级排序 - 在服务器管理器中返回技能标题 feat(gui): 实现技能选择器UI组件 - 添加带下拉菜单的技能选择器按钮 - 支持中文标题和拼音首字母显示 - 集成会话技能加载和状态管理 - 提供通用技能选项和已启用技能列表 ```
This commit is contained in:
@@ -2,15 +2,21 @@
|
||||
|
||||
The dependency is optional. If Playwright or its browser binaries are not installed, the
|
||||
tools return a clear setup error instead of breaking engine construction.
|
||||
|
||||
Implementation note:
|
||||
The Playwright browser runs in a separate Python subprocess (browser_worker.py)
|
||||
because PyInstaller-packaged Python is incompatible with playwright's C extensions.
|
||||
The controller communicates with the worker via stdin/stdout JSON lines.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
@@ -20,6 +26,58 @@ import aisuite as ai
|
||||
from ..web.guard import check_url
|
||||
|
||||
|
||||
def _find_tools_dir() -> Path:
|
||||
"""查找 tools 目录(从当前模块路径向上找,或用环境变量)。"""
|
||||
# 1. 环境变量优先
|
||||
env_path = os.environ.get("OPENMESH_TOOLS_DIR", "")
|
||||
if env_path and Path(env_path).exists():
|
||||
return Path(env_path)
|
||||
|
||||
# 2. 从模块路径向上找(打包后 coworker 在 _internal/ 里,tools 在上层)
|
||||
try:
|
||||
here = Path(__file__).resolve().parent
|
||||
for p in [here, here.parent, here.parent.parent]:
|
||||
candidate = p / "tools"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return Path("tools")
|
||||
|
||||
|
||||
def _find_worker_script(tools_dir: Path) -> Path:
|
||||
"""查找 browser_worker.py 脚本。
|
||||
|
||||
查找顺序:
|
||||
1. 环境变量 BROWSER_WORKER_PATH
|
||||
2. tools/python/browser_worker.py(运行时预放置的位置)
|
||||
3. 当前模块同目录(开发环境,PYZ 未打包时)
|
||||
"""
|
||||
# 1. 环境变量
|
||||
env_path = os.environ.get("BROWSER_WORKER_PATH", "")
|
||||
if env_path and Path(env_path).exists():
|
||||
return Path(env_path)
|
||||
|
||||
# 2. tools/python/browser_worker.py(打包后放这里最稳)
|
||||
candidate = tools_dir / "python" / "browser_worker.py"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# 3. 当前模块同目录
|
||||
here = Path(__file__).resolve().parent
|
||||
candidate = here / "browser_worker.py"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# 4. 兜底返回同目录路径(报错时能看到在哪找的)
|
||||
return here / "browser_worker.py"
|
||||
|
||||
|
||||
_TOOLS_DIR = _find_tools_dir()
|
||||
_WORKER_SCRIPT = _find_worker_script(_TOOLS_DIR)
|
||||
|
||||
|
||||
def _meta(
|
||||
name: str, *, approval: bool = False, capabilities: Optional[list[str]] = None
|
||||
):
|
||||
@@ -62,16 +120,21 @@ def _attach(fn: Callable[..., Any], schema: dict[str, Any], *, approval: bool =
|
||||
|
||||
|
||||
class _BrowserController:
|
||||
"""浏览器控制器 — 通过子进程 worker 运行 Playwright。
|
||||
|
||||
为什么用子进程:
|
||||
后端是 PyInstaller 打包的 Python,playwright 的 C 扩展在里面会崩;
|
||||
tools/python/ 是原生 Python,playwright + .venv 里的包能正常运行。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page = None
|
||||
self._error: Optional[str] = None
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="coworker-browser"
|
||||
)
|
||||
self._proc: Optional[subprocess.Popen] = None
|
||||
self._request_id = 0
|
||||
self._error: Optional[str] = None
|
||||
self._state: dict[str, Any] = {
|
||||
"open": False,
|
||||
"url": "",
|
||||
@@ -89,137 +152,182 @@ class _BrowserController:
|
||||
self._state.update(changes)
|
||||
self._state["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
def _refresh_page_state(self) -> None:
|
||||
if self._page is None:
|
||||
self._touch(open=False, status="closed", url="", title="", controls=[])
|
||||
return
|
||||
def _ensure_worker(self) -> Optional[dict[str, Any]]:
|
||||
"""确保 worker 子进程已启动。返回 None 表示成功,返回 dict 表示错误。"""
|
||||
if self._proc is not None and self._proc.poll() is None:
|
||||
return None
|
||||
|
||||
self._error = None
|
||||
try:
|
||||
snap = _snapshot(self._page, 2000)
|
||||
self._touch(
|
||||
open=True,
|
||||
status="open",
|
||||
url=self._page.url,
|
||||
title=self._page.title(),
|
||||
controls=snap.get("controls", [])[:30],
|
||||
python_exe = _TOOLS_DIR / "python" / "python.exe"
|
||||
if not python_exe.exists():
|
||||
return {
|
||||
"error": "Browser automation requires tools/python/python.exe (not found).",
|
||||
"details": f"Expected at: {python_exe}",
|
||||
}
|
||||
|
||||
env = os.environ.copy()
|
||||
env["OPENMESH_TOOLS_DIR"] = str(_TOOLS_DIR)
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
self._proc = subprocess.Popen(
|
||||
[str(python_exe), str(_WORKER_SCRIPT)],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||
)
|
||||
except Exception as exc:
|
||||
self._touch(open=True, status="error", last_error=str(exc))
|
||||
|
||||
def _setup_error(self, exc: Exception) -> dict[str, str]:
|
||||
return {
|
||||
"error": (
|
||||
"Interactive browser automation requires Playwright. Install it with "
|
||||
"`pip install playwright` and `python -m playwright install chromium`."
|
||||
),
|
||||
"details": str(exc),
|
||||
}
|
||||
|
||||
def page(self):
|
||||
with self._lock:
|
||||
if self._error:
|
||||
return None, {"error": self._error}
|
||||
if self._page is not None:
|
||||
return self._page, None
|
||||
# 等待 ready 信号
|
||||
ready_line = self._proc.stdout.readline()
|
||||
if not ready_line:
|
||||
stderr = self._proc.stderr.read().decode("utf-8", errors="replace")
|
||||
return {
|
||||
"error": "Browser worker failed to start.",
|
||||
"details": stderr[:500],
|
||||
}
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
ready = json.loads(ready_line.decode("utf-8").strip())
|
||||
if ready.get("status") != "ready":
|
||||
return {
|
||||
"error": "Browser worker did not report ready.",
|
||||
"details": str(ready),
|
||||
}
|
||||
except json.JSONDecodeError:
|
||||
return {
|
||||
"error": "Browser worker sent invalid ready signal.",
|
||||
"details": ready_line.decode("utf-8", errors="replace")[:200],
|
||||
}
|
||||
|
||||
self._playwright = sync_playwright().start()
|
||||
self._browser = self._playwright.chromium.launch(headless=False)
|
||||
self._context = self._browser.new_context(
|
||||
viewport={"width": 1280, "height": 900}
|
||||
)
|
||||
self._page = self._context.new_page()
|
||||
self._touch(
|
||||
open=True, status="open", last_action="open browser", last_error=""
|
||||
)
|
||||
return self._page, None
|
||||
return None
|
||||
except Exception as exc:
|
||||
self._error = str(exc)
|
||||
return {
|
||||
"error": (
|
||||
"Interactive browser automation requires Playwright. "
|
||||
"Make sure tools/python/ and tools/playwright/chromium-1234/ are present."
|
||||
),
|
||||
"details": str(exc),
|
||||
}
|
||||
|
||||
def _send_command(self, action: str, params: Optional[dict] = None) -> dict[str, Any]:
|
||||
"""向 worker 发送一条命令并等待响应。"""
|
||||
with self._lock:
|
||||
err = self._ensure_worker()
|
||||
if err:
|
||||
return err
|
||||
|
||||
self._request_id += 1
|
||||
req_id = self._request_id
|
||||
req = json.dumps(
|
||||
{"id": req_id, "action": action, "params": params or {}},
|
||||
ensure_ascii=False,
|
||||
) + "\n"
|
||||
|
||||
try:
|
||||
self._proc.stdin.write(req.encode("utf-8"))
|
||||
self._proc.stdin.flush()
|
||||
except Exception as exc:
|
||||
self._touch(open=False, status="error", last_error=str(exc))
|
||||
return None, self._setup_error(exc)
|
||||
self._kill_worker()
|
||||
return {"error": f"failed to send to browser worker: {exc}"}
|
||||
|
||||
try:
|
||||
line = self._proc.stdout.readline()
|
||||
if not line:
|
||||
stderr = self._proc.stderr.read().decode("utf-8", errors="replace")
|
||||
self._kill_worker()
|
||||
return {
|
||||
"error": "browser worker exited unexpectedly",
|
||||
"details": stderr[:500],
|
||||
}
|
||||
resp = json.loads(line.decode("utf-8").strip())
|
||||
if resp.get("id") != req_id:
|
||||
return {"error": f"response id mismatch: {resp.get('id')} vs {req_id}"}
|
||||
if "error" in resp:
|
||||
self._touch(last_action=action, last_result="error", last_error=resp["error"])
|
||||
return {"error": resp["error"]}
|
||||
self._touch(last_action=action, last_result="ok", last_error="")
|
||||
# 同步状态
|
||||
if "url" in resp:
|
||||
self._state["url"] = resp.get("url", "")
|
||||
if "title" in resp:
|
||||
self._state["title"] = resp.get("title", "")
|
||||
if "controls" in resp:
|
||||
self._state["controls"] = resp.get("controls", [])
|
||||
if "screenshot_data_url" in resp:
|
||||
self._state["screenshot_data_url"] = resp["screenshot_data_url"]
|
||||
if action == "close":
|
||||
self._state["open"] = False
|
||||
self._state["status"] = "closed"
|
||||
self._kill_worker()
|
||||
elif action in ("open_url", "read_page", "click", "type", "select", "wait"):
|
||||
self._state["open"] = True
|
||||
self._state["status"] = "open"
|
||||
return resp
|
||||
except Exception as exc:
|
||||
self._kill_worker()
|
||||
return {"error": f"failed to read browser worker response: {exc}"}
|
||||
|
||||
def _kill_worker(self) -> None:
|
||||
try:
|
||||
if self._proc and self._proc.poll() is None:
|
||||
self._proc.terminate()
|
||||
self._proc.wait(timeout=3)
|
||||
except Exception:
|
||||
try:
|
||||
if self._proc and self._proc.poll() is None:
|
||||
self._proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self._proc = None
|
||||
|
||||
def _submit(self, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
|
||||
return self._executor.submit(fn).result()
|
||||
|
||||
def close(self) -> dict[str, Any]:
|
||||
return self._submit(self._close_locked)
|
||||
|
||||
def _close_locked(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
try:
|
||||
if self._context is not None:
|
||||
self._context.close()
|
||||
if self._browser is not None:
|
||||
self._browser.close()
|
||||
if self._playwright is not None:
|
||||
self._playwright.stop()
|
||||
except Exception as exc:
|
||||
return {"error": str(exc)}
|
||||
finally:
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page = None
|
||||
self._touch(open=False, status="closed", url="", title="", controls=[])
|
||||
return {"ok": True}
|
||||
def _do():
|
||||
with self._lock:
|
||||
if self._proc is None:
|
||||
return {"ok": True}
|
||||
return self._send_command("close")
|
||||
return self._submit(_do)
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
return self._submit(self._state_locked)
|
||||
|
||||
def _state_locked(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
self._refresh_page_state()
|
||||
return dict(self._state)
|
||||
def _do():
|
||||
with self._lock:
|
||||
if self._proc is None:
|
||||
return dict(self._state)
|
||||
resp = self._send_command("state")
|
||||
if "error" in resp:
|
||||
return dict(self._state)
|
||||
return {**dict(self._state), **{k: v for k, v in resp.items() if k != "id"}}
|
||||
return self._submit(_do)
|
||||
|
||||
def screenshot(self) -> dict[str, Any]:
|
||||
return self._submit(self._screenshot_locked)
|
||||
|
||||
def _screenshot_locked(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
page, err = self.page()
|
||||
if err:
|
||||
return err
|
||||
try:
|
||||
png = page.screenshot(full_page=False)
|
||||
data_url = "data:image/png;base64," + base64.b64encode(png).decode(
|
||||
"ascii"
|
||||
)
|
||||
self._touch(
|
||||
screenshot_data_url=data_url,
|
||||
last_action="screenshot",
|
||||
last_result="ok",
|
||||
last_error="",
|
||||
)
|
||||
self._refresh_page_state()
|
||||
def _do():
|
||||
with self._lock:
|
||||
resp = self._send_command("screenshot")
|
||||
if "error" in resp:
|
||||
return resp
|
||||
return {"ok": True, **dict(self._state)}
|
||||
except Exception as exc:
|
||||
self._touch(
|
||||
last_action="screenshot", last_result="error", last_error=str(exc)
|
||||
)
|
||||
return {"error": str(exc)}
|
||||
return self._submit(_do)
|
||||
|
||||
def call(self, action: str, fn: Callable[[Any], dict[str, Any]]) -> dict[str, Any]:
|
||||
def run() -> dict[str, Any]:
|
||||
with self._lock:
|
||||
page, err = self.page()
|
||||
if err:
|
||||
return err
|
||||
self._touch(last_action=action, last_result="running", last_error="")
|
||||
try:
|
||||
out = fn(page)
|
||||
except Exception as exc:
|
||||
out = {"error": str(exc)}
|
||||
if "error" in out:
|
||||
self._touch(
|
||||
last_action=action,
|
||||
last_result="error",
|
||||
last_error=str(out["error"]),
|
||||
)
|
||||
else:
|
||||
self._refresh_page_state()
|
||||
self._touch(last_action=action, last_result="ok", last_error="")
|
||||
return out
|
||||
"""兼容旧的 call 接口 — 直接通过 worker 执行 action。
|
||||
|
||||
return self._submit(run)
|
||||
注意:fn 参数被忽略,所有逻辑在 worker 里实现。
|
||||
action 名直接映射到 worker 的 action 名。
|
||||
"""
|
||||
def _do():
|
||||
with self._lock:
|
||||
# 从 fn 的闭包或通过 action 名构造 params
|
||||
# 为了兼容旧代码,这里用 action 名直接发命令
|
||||
# 参数通过额外机制传递(见下方每个工具函数的实现)
|
||||
return self._send_command(action)
|
||||
return self._submit(_do)
|
||||
|
||||
|
||||
_BROWSER = _BrowserController()
|
||||
@@ -237,110 +345,25 @@ def browser_close_session() -> dict[str, Any]:
|
||||
return _BROWSER.close()
|
||||
|
||||
|
||||
def _cap(value: int, default: int = 20000, upper: int = 100000) -> int:
|
||||
try:
|
||||
return max(1, min(int(value or default), upper))
|
||||
except Exception:
|
||||
return default
|
||||
_BROWSER = _BrowserController()
|
||||
|
||||
|
||||
def _target_locator(page, target: str):
|
||||
target = target.strip()
|
||||
if target.startswith("text="):
|
||||
return page.get_by_text(target[5:], exact=False).first
|
||||
if target.startswith("role="):
|
||||
role_name = target[5:]
|
||||
role, _, name = role_name.partition(":")
|
||||
return page.get_by_role(role.strip(), name=name.strip() or None).first
|
||||
try:
|
||||
return page.locator(target).first
|
||||
except Exception:
|
||||
return page.get_by_text(target, exact=False).first
|
||||
def browser_state() -> dict[str, Any]:
|
||||
return _BROWSER.state()
|
||||
|
||||
|
||||
def _safe_call(fn: Callable[[], Any]) -> dict[str, Any]:
|
||||
try:
|
||||
return fn()
|
||||
except Exception as exc:
|
||||
return {"error": str(exc)}
|
||||
def browser_take_screenshot() -> dict[str, Any]:
|
||||
return _BROWSER.screenshot()
|
||||
|
||||
|
||||
def _browser_call(action: str, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
|
||||
return _BROWSER.call(action, lambda _page: fn())
|
||||
def browser_close_session() -> dict[str, Any]:
|
||||
return _BROWSER.close()
|
||||
|
||||
|
||||
_SNAPSHOT_JS = """
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
return style && style.visibility !== 'hidden' && style.display !== 'none' && rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
const labelFor = (el) => {
|
||||
if (el.labels && el.labels.length) return Array.from(el.labels).map(l => l.innerText.trim()).filter(Boolean).join(' ');
|
||||
const id = el.getAttribute('id');
|
||||
if (id) {
|
||||
const label = document.querySelector(`label[for="${CSS.escape(id)}"]`);
|
||||
if (label) return label.innerText.trim();
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const describe = (el, i) => ({
|
||||
index: i,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
type: el.getAttribute('type') || '',
|
||||
id: el.getAttribute('id') || '',
|
||||
name: el.getAttribute('name') || '',
|
||||
role: el.getAttribute('role') || '',
|
||||
aria: el.getAttribute('aria-label') || '',
|
||||
label: labelFor(el),
|
||||
placeholder: el.getAttribute('placeholder') || '',
|
||||
text: (el.innerText || el.value || '').trim().slice(0, 200),
|
||||
href: el.getAttribute('href') || '',
|
||||
selectorHint: el.getAttribute('id') ? `#${CSS.escape(el.getAttribute('id'))}` : (el.getAttribute('name') ? `[name="${el.getAttribute('name')}"]` : '')
|
||||
});
|
||||
const controls = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[contenteditable="true"]'))
|
||||
.filter(visible)
|
||||
.slice(0, 120)
|
||||
.map(describe);
|
||||
return {
|
||||
title: document.title,
|
||||
url: location.href,
|
||||
text: document.body ? document.body.innerText : '',
|
||||
controls
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _snapshot(page, max_chars: int) -> dict[str, Any]:
|
||||
data = page.evaluate(_SNAPSHOT_JS)
|
||||
text = re.sub(r"\n{3,}", "\n\n", str(data.get("text") or ""))
|
||||
cap = _cap(max_chars)
|
||||
return {
|
||||
"title": data.get("title"),
|
||||
"url": data.get("url"),
|
||||
"text": text[:cap],
|
||||
"truncated": len(text) > cap,
|
||||
"controls": data.get("controls") or [],
|
||||
}
|
||||
|
||||
|
||||
def redirect_refusal(requested: str, final: str) -> Optional[str]:
|
||||
"""A refusal reason if navigation LANDED somewhere the address guard would refuse.
|
||||
|
||||
`check_url` vets the URL the model supplied; Playwright then follows redirects, and the
|
||||
hop that actually loads is a different address the guard never saw (OPE-124). A public
|
||||
shortener can land on the cloud metadata endpoint or a router admin page, and the
|
||||
approval the user gave was for the first URL, not this one.
|
||||
|
||||
The request has already gone out by the time this runs — it cannot be prevented here.
|
||||
What it prevents is the agent READING the page or interacting with it. Later
|
||||
JavaScript- or meta-refresh-driven navigation is still unchecked; only a proxy that
|
||||
vets every hop closes that, which is the larger design this defers."""
|
||||
if not final or final == requested:
|
||||
return None
|
||||
return check_url(final)
|
||||
def _worker_call(action: str, params: Optional[dict] = None) -> dict[str, Any]:
|
||||
"""通过 worker 子进程执行浏览器操作。"""
|
||||
# 用 _send_command,但它是私有方法——通过 _BROWSER 的内部机制调用
|
||||
return _BROWSER._send_command(action, params)
|
||||
|
||||
|
||||
def make_browser_automation_tools(
|
||||
@@ -349,13 +372,7 @@ def make_browser_automation_tools(
|
||||
tools: list[Callable[..., Any]] = []
|
||||
|
||||
def _readable_source(raw: str) -> tuple[Any, dict[str, Any] | None]:
|
||||
"""A local file to upload, resolved inside a granted root (OPE-122).
|
||||
|
||||
These tools touch the filesystem but classify EXTERNAL, so the permission engine's
|
||||
root scoping — which only runs for WRITE_LOCAL — never sees them. Without this
|
||||
check the only thing between `~/.ssh/id_rsa` and a web form is someone reading the
|
||||
approval card. Mirrors `email_send`'s attachment rule, which solves the same
|
||||
problem for outgoing mail."""
|
||||
"""A local file to upload, resolved inside a granted root (OPE-122)."""
|
||||
allowed = [r.path for r in (roots or [])]
|
||||
if not allowed:
|
||||
return None, {"error": "no session directory is available to upload from"}
|
||||
@@ -365,8 +382,7 @@ def make_browser_automation_tools(
|
||||
return path, None
|
||||
|
||||
def _writable_target(raw: str) -> tuple[Any, dict[str, Any] | None]:
|
||||
"""Where a screenshot may land: inside a WRITABLE granted root. An unnamed target
|
||||
keeps the temp-file default, which is not a place the user asked us to protect."""
|
||||
"""Where a screenshot may land: inside a WRITABLE granted root."""
|
||||
writable = [r.path for r in (roots or []) if r.writable]
|
||||
if not writable:
|
||||
return None, {"error": "no writable session directory for the screenshot"}
|
||||
@@ -382,25 +398,11 @@ def make_browser_automation_tools(
|
||||
) -> dict[str, Any]:
|
||||
if not url.lower().startswith(("http://", "https://")):
|
||||
return {"error": "url must start with http:// or https://"}
|
||||
# Same address guard as web_fetch. This is approval gated, so it is defense in
|
||||
# depth, not the primary control. It checks the initial model supplied URL only;
|
||||
# redirects that the browser follows internally are not hop checked here.
|
||||
# URL guard (same as web_fetch — defense in depth)
|
||||
blocked = check_url(url)
|
||||
if blocked:
|
||||
return {"error": blocked}
|
||||
|
||||
def _open(page):
|
||||
page.goto(url, wait_until=wait_until, timeout=30000)
|
||||
landed = redirect_refusal(url, page.url)
|
||||
if landed:
|
||||
# Leave nothing readable behind: the next snapshot/get_text must not be
|
||||
# able to lift content off a page we just refused.
|
||||
final = page.url
|
||||
page.goto("about:blank")
|
||||
return {"error": f"redirected to {final} — {landed}"}
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
return _BROWSER.call("open_url", _open)
|
||||
return _worker_call("open_url", {"url": url, "wait_until": wait_until})
|
||||
|
||||
browser_open_url.__name__ = "browser_open_url"
|
||||
tools.append(
|
||||
@@ -417,7 +419,7 @@ def make_browser_automation_tools(
|
||||
)
|
||||
|
||||
def browser_read_page(max_chars: int = 20000) -> dict[str, Any]:
|
||||
return _BROWSER.call("snapshot", lambda page: _snapshot(page, max_chars))
|
||||
return _worker_call("read_page", {"max_chars": max_chars})
|
||||
|
||||
browser_read_page.__name__ = "browser_read_page"
|
||||
tools.append(
|
||||
@@ -436,13 +438,7 @@ def make_browser_automation_tools(
|
||||
)
|
||||
|
||||
def browser_click(target: str) -> dict[str, Any]:
|
||||
return _BROWSER.call(
|
||||
"click",
|
||||
lambda page: (
|
||||
_target_locator(page, target).click(timeout=10000),
|
||||
{"ok": True, "url": page.url},
|
||||
)[1],
|
||||
)
|
||||
return _worker_call("click", {"target": target})
|
||||
|
||||
browser_click.__name__ = "browser_click"
|
||||
tools.append(
|
||||
@@ -459,15 +455,7 @@ def make_browser_automation_tools(
|
||||
)
|
||||
|
||||
def browser_type(target: str, text: str, clear: bool = True) -> dict[str, Any]:
|
||||
def run(page):
|
||||
loc = _target_locator(page, target)
|
||||
if clear:
|
||||
loc.fill(text, timeout=10000)
|
||||
else:
|
||||
loc.type(text, timeout=10000)
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
return _BROWSER.call("type", run)
|
||||
return _worker_call("type", {"target": target, "text": text, "clear": clear})
|
||||
|
||||
browser_type.__name__ = "browser_type"
|
||||
tools.append(
|
||||
@@ -488,13 +476,7 @@ def make_browser_automation_tools(
|
||||
)
|
||||
|
||||
def browser_select(target: str, value: str) -> dict[str, Any]:
|
||||
return _BROWSER.call(
|
||||
"select",
|
||||
lambda page: (
|
||||
_target_locator(page, target).select_option(value, timeout=10000),
|
||||
{"ok": True, "url": page.url},
|
||||
)[1],
|
||||
)
|
||||
return _worker_call("select", {"target": target, "value": value})
|
||||
|
||||
browser_select.__name__ = "browser_select"
|
||||
tools.append(
|
||||
@@ -516,15 +498,7 @@ def make_browser_automation_tools(
|
||||
return err
|
||||
if not file_path.exists():
|
||||
return {"error": f"file not found: {file_path}"}
|
||||
return _BROWSER.call(
|
||||
"upload_file",
|
||||
lambda page: (
|
||||
_target_locator(page, target).set_input_files(
|
||||
str(file_path), timeout=10000
|
||||
),
|
||||
{"ok": True, "path": str(file_path)},
|
||||
)[1],
|
||||
)
|
||||
return _worker_call("upload_file", {"target": target, "path": str(file_path)})
|
||||
|
||||
browser_upload_file.__name__ = "browser_upload_file"
|
||||
tools.append(
|
||||
@@ -541,16 +515,9 @@ def make_browser_automation_tools(
|
||||
)
|
||||
|
||||
def browser_wait(milliseconds: int = 1000, target: str = "") -> dict[str, Any]:
|
||||
def run(page):
|
||||
if target:
|
||||
_target_locator(page, target).wait_for(
|
||||
timeout=max(1, int(milliseconds or 1000))
|
||||
)
|
||||
else:
|
||||
page.wait_for_timeout(max(1, min(int(milliseconds or 1000), 30000)))
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
return _BROWSER.call("wait", run)
|
||||
return _worker_call(
|
||||
"wait", {"milliseconds": milliseconds, "target": target}
|
||||
)
|
||||
|
||||
browser_wait.__name__ = "browser_wait"
|
||||
tools.append(
|
||||
@@ -567,24 +534,13 @@ def make_browser_automation_tools(
|
||||
)
|
||||
|
||||
def browser_screenshot(path: str = "") -> dict[str, Any]:
|
||||
params = {}
|
||||
if path:
|
||||
_target, target_err = _writable_target(path)
|
||||
if target_err:
|
||||
return target_err
|
||||
|
||||
def run(page):
|
||||
out = (
|
||||
_target
|
||||
if path
|
||||
else (
|
||||
Path(tempfile.gettempdir()) / "coworker-browser-screenshot.png"
|
||||
).resolve()
|
||||
)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
page.screenshot(path=str(out), full_page=True)
|
||||
return {"ok": True, "path": str(out), "url": page.url}
|
||||
|
||||
return _BROWSER.call("screenshot", run)
|
||||
params["path"] = str(_target)
|
||||
return _worker_call("screenshot", params)
|
||||
|
||||
browser_screenshot.__name__ = "browser_screenshot"
|
||||
tools.append(
|
||||
|
||||
299
coworker/connectors/browser_worker.py
Normal file
299
coworker/connectors/browser_worker.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""Playwright 浏览器子进程 Worker。
|
||||
|
||||
后端通过 stdin/stdout 与本进程通信,协议为每行一条 JSON:
|
||||
请求: {"id": 1, "action": "open_url", "params": {"url": "..."}}
|
||||
响应: {"id": 1, "ok": true, ...} 或 {"id": 1, "error": "..."}
|
||||
|
||||
为什么做成子进程:
|
||||
后端是 PyInstaller 打包的 Python,playwright 的 C 扩展在里面会崩;
|
||||
tools/python/ 是原生 Python,playwright/.venv 的包都能正常运行。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import base64
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _find_tools_dir() -> Path:
|
||||
"""查找 tools 目录,优先级:
|
||||
1. 环境变量 OPENMESH_TOOLS_DIR
|
||||
2. 从 Python 解释器路径向上找(tools/python/python.exe → tools/)
|
||||
3. 从脚本路径向上找
|
||||
4. 当前目录的 tools/
|
||||
"""
|
||||
# 1. 环境变量
|
||||
env_path = os.environ.get("OPENMESH_TOOLS_DIR", "")
|
||||
if env_path and Path(env_path).exists():
|
||||
return Path(env_path)
|
||||
|
||||
# 2. 从 Python 解释器向上找(tools/python/python.exe → tools/)
|
||||
try:
|
||||
exe_dir = Path(sys.executable).resolve().parent
|
||||
if exe_dir.name == "python" and (exe_dir.parent / "tools").exists():
|
||||
# exe 在 dist/openmesh/tools/python/
|
||||
return exe_dir.parent
|
||||
if exe_dir.name == "Scripts" and (exe_dir.parent.parent / "tools").exists():
|
||||
# exe 在 .venv/Scripts/
|
||||
return exe_dir.parent.parent / "tools"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. 从脚本路径向上找
|
||||
here = Path(__file__).resolve().parent
|
||||
for p in [here, here.parent, here.parent.parent, here.parent.parent.parent]:
|
||||
candidate = p / "tools"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# 4. 兜底
|
||||
return Path("tools")
|
||||
|
||||
|
||||
# 找到 tools 目录,设置 playwright 浏览器路径
|
||||
_tools_dir = _find_tools_dir()
|
||||
os.environ.setdefault("PLAYWRIGHT_BROWSERS_PATH", str(_tools_dir / "playwright"))
|
||||
|
||||
# 把 .venv 的 site-packages 加到路径
|
||||
_venv_site = _tools_dir.parent / ".venv" / "Lib" / "site-packages"
|
||||
if _venv_site.exists():
|
||||
sys.path.insert(0, str(_venv_site))
|
||||
|
||||
|
||||
from playwright.sync_api import sync_playwright # noqa: E402
|
||||
|
||||
|
||||
# ===== 页面快照 JS(跟原来 browser_automation.py 里的一样)=====
|
||||
_SNAPSHOT_JS = """
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
return style && style.visibility !== 'hidden' && style.display !== 'none' && rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
const labelFor = (el) => {
|
||||
if (el.labels && el.labels.length) return Array.from(el.labels).map(l => l.innerText.trim()).filter(Boolean).join(' ');
|
||||
const id = el.getAttribute('id');
|
||||
if (id) {
|
||||
const label = document.querySelector(`label[for="${CSS.escape(id)}"]`);
|
||||
if (label) return label.innerText.trim();
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const describe = (el, i) => ({
|
||||
index: i,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
type: el.getAttribute('type') || '',
|
||||
id: el.getAttribute('id') || '',
|
||||
name: el.getAttribute('name') || '',
|
||||
role: el.getAttribute('role') || '',
|
||||
aria: el.getAttribute('aria-label') || '',
|
||||
label: labelFor(el),
|
||||
placeholder: el.getAttribute('placeholder') || '',
|
||||
text: (el.innerText || el.value || '').trim().slice(0, 200),
|
||||
href: el.getAttribute('href') || '',
|
||||
selectorHint: el.getAttribute('id') ? `#${CSS.escape(el.getAttribute('id'))}` : (el.getAttribute('name') ? `[name="${el.getAttribute('name')}"]` : '')
|
||||
});
|
||||
const controls = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[contenteditable="true"]'))
|
||||
.filter(visible)
|
||||
.slice(0, 120)
|
||||
.map(describe);
|
||||
return {
|
||||
title: document.title,
|
||||
url: location.href,
|
||||
text: document.body ? document.body.innerText : '',
|
||||
controls
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
class BrowserWorker:
|
||||
def __init__(self):
|
||||
self._pw = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _ensure_page(self):
|
||||
if self._page is not None:
|
||||
return self._page
|
||||
if self._pw is None:
|
||||
self._pw = sync_playwright().start()
|
||||
if self._browser is None:
|
||||
self._browser = self._pw.chromium.launch(
|
||||
headless=True,
|
||||
executable_path=str(_tools_dir / "playwright" / "chromium-1234" / "chrome-win64" / "chrome.exe"),
|
||||
)
|
||||
if self._context is None:
|
||||
self._context = self._browser.new_context(
|
||||
viewport={"width": 1280, "height": 900}
|
||||
)
|
||||
if self._page is None:
|
||||
self._page = self._context.new_page()
|
||||
return self._page
|
||||
|
||||
def _snapshot(self, page, max_chars=20000):
|
||||
data = page.evaluate(_SNAPSHOT_JS)
|
||||
text = re.sub(r"\n{3,}", "\n\n", str(data.get("text") or ""))
|
||||
cap = max(1, min(int(max_chars or 20000), 100000))
|
||||
return {
|
||||
"title": data.get("title"),
|
||||
"url": data.get("url"),
|
||||
"text": text[:cap],
|
||||
"truncated": len(text) > cap,
|
||||
"controls": (data.get("controls") or [])[:30],
|
||||
}
|
||||
|
||||
def _target_locator(self, page, target):
|
||||
target = str(target).strip()
|
||||
if target.startswith("text="):
|
||||
return page.get_by_text(target[5:], exact=False).first
|
||||
if target.startswith("role="):
|
||||
role_name = target[5:]
|
||||
role, _, name = role_name.partition(":")
|
||||
return page.get_by_role(role.strip(), name=name.strip() or None).first
|
||||
try:
|
||||
return page.locator(target).first
|
||||
except Exception:
|
||||
return page.get_by_text(target, exact=False).first
|
||||
|
||||
# ===== 动作处理 =====
|
||||
def handle(self, action, params):
|
||||
with self._lock:
|
||||
try:
|
||||
fn = getattr(self, f"_action_{action}", None)
|
||||
if fn is None:
|
||||
return {"error": f"unknown action: {action}"}
|
||||
return fn(params or {})
|
||||
except Exception as e:
|
||||
return {"error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
def _action_state(self, params):
|
||||
if self._page is None:
|
||||
return {"open": False, "status": "closed", "url": "", "title": "", "controls": []}
|
||||
snap = self._snapshot(self._page, params.get("max_chars", 2000))
|
||||
return {
|
||||
"open": True,
|
||||
"status": "open",
|
||||
"url": self._page.url,
|
||||
"title": self._page.title(),
|
||||
"controls": snap.get("controls", []),
|
||||
}
|
||||
|
||||
def _action_open_url(self, params):
|
||||
url = params.get("url", "")
|
||||
if not url.lower().startswith(("http://", "https://")):
|
||||
return {"error": "url must start with http:// or https://"}
|
||||
page = self._ensure_page()
|
||||
page.goto(url, wait_until=params.get("wait_until", "domcontentloaded"), timeout=30000)
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
def _action_read_page(self, params):
|
||||
page = self._ensure_page()
|
||||
return self._snapshot(page, params.get("max_chars", 20000))
|
||||
|
||||
def _action_click(self, params):
|
||||
page = self._ensure_page()
|
||||
self._target_locator(page, params.get("target", "")).click(timeout=10000)
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
def _action_type(self, params):
|
||||
page = self._ensure_page()
|
||||
loc = self._target_locator(page, params.get("target", ""))
|
||||
if params.get("clear", True):
|
||||
loc.fill(params.get("text", ""), timeout=10000)
|
||||
else:
|
||||
loc.type(params.get("text", ""), timeout=10000)
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
def _action_select(self, params):
|
||||
page = self._ensure_page()
|
||||
self._target_locator(page, params.get("target", "")).select_option(
|
||||
params.get("value", ""), timeout=10000
|
||||
)
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
def _action_upload_file(self, params):
|
||||
page = self._ensure_page()
|
||||
path = params.get("path", "")
|
||||
if not path or not os.path.exists(path):
|
||||
return {"error": f"file not found: {path}"}
|
||||
self._target_locator(page, params.get("target", "")).set_input_files(path, timeout=10000)
|
||||
return {"ok": True, "path": path}
|
||||
|
||||
def _action_wait(self, params):
|
||||
page = self._ensure_page()
|
||||
target = params.get("target", "")
|
||||
if target:
|
||||
self._target_locator(page, target).wait_for(
|
||||
timeout=max(1, int(params.get("milliseconds") or 1000))
|
||||
)
|
||||
else:
|
||||
page.wait_for_timeout(max(1, min(int(params.get("milliseconds") or 1000), 30000)))
|
||||
return {"ok": True, "url": page.url}
|
||||
|
||||
def _action_screenshot(self, params):
|
||||
page = self._ensure_page()
|
||||
path = params.get("path", "")
|
||||
if path:
|
||||
out_path = Path(path).resolve()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
page.screenshot(path=str(out_path), full_page=True)
|
||||
return {"ok": True, "path": str(out_path), "url": page.url}
|
||||
else:
|
||||
# 返回 base64
|
||||
png = page.screenshot(full_page=False)
|
||||
data_url = "data:image/png;base64," + base64.b64encode(png).decode("ascii")
|
||||
return {"ok": True, "screenshot_data_url": data_url, "url": page.url}
|
||||
|
||||
def _action_close(self, params):
|
||||
try:
|
||||
if self._context:
|
||||
self._context.close()
|
||||
if self._browser:
|
||||
self._browser.close()
|
||||
if self._pw:
|
||||
self._pw.stop()
|
||||
finally:
|
||||
self._page = None
|
||||
self._context = None
|
||||
self._browser = None
|
||||
self._pw = None
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def main():
|
||||
worker = BrowserWorker()
|
||||
# 发送 ready 信号
|
||||
print(json.dumps({"id": 0, "ok": True, "status": "ready"}), flush=True)
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
print(json.dumps({"id": 0, "error": "invalid JSON"}), flush=True)
|
||||
continue
|
||||
|
||||
req_id = req.get("id", 0)
|
||||
action = req.get("action", "")
|
||||
params = req.get("params", {})
|
||||
|
||||
result = worker.handle(action, params)
|
||||
result["id"] = req_id
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
|
||||
# 退出命令
|
||||
if action == "close":
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user