senmeshs/OpenMesh
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

```
Some checks failed
CI / pytest (push) Has been cancelled
Details
CI / gui-unit (push) Has been cancelled
Details
CI / gui-e2e (push) Has been cancelled
Details

Browse Source 
feat(agent): 添加技能自动路由功能
- 引入 SkillRouter 实现根据用户消息自动推荐技能
- 在构建引擎时集成技能路由器
- 从对话历史中提取最后一条用户消息作为路由输入
- 为技能添加触发关键词和中文标题字段支持

refactor(browser): 重构浏览器自动化为子进程架构

- 将 Playwright 浏览器控制移至独立的子进程 worker
- 解决 PyInstaller 打包环境下 C 扩展兼容性问题
- 通过 JSON RPC 协议与浏览器 worker 通信
- 添加工具目录和脚本路径查找机制

feat(skills): 增强技能元数据和UI展示

- 为技能添加 triggers 和 title 字段
- 在技能商店中包含中文标题信息
- 添加 office-viz 技能优先级排序
- 在服务器管理器中返回技能标题

feat(gui): 实现技能选择器UI组件

- 添加带下拉菜单的技能选择器按钮
- 支持中文标题和拼音首字母显示
- 集成会话技能加载和状态管理
- 提供通用技能选项和已启用技能列表
```
This commit is contained in:
zhaolei
2026-09-14 17:36:02 +08:00
parent 6f402ffcee
commit 3e356b117a
10 changed files with 1493 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

550
coworker/connectors/browser_automation.py
View File

@@ -2,15 +2,21 @@
The dependency is optional. If Playwright or its browser binaries are not installed, the
tools return a clear setup error instead of breaking engine construction.
Implementation note:
The Playwright browser runs in a separate Python subprocess (browser_worker.py)
because PyInstaller-packaged Python is incompatible with playwright's C extensions.
The controller communicates with the worker via stdin/stdout JSON lines.
"""
from __future__ import annotations
import re
import json
import os
import subprocess
import tempfile
import threading
import time
import base64
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, Callable, Optional
@@ -20,6 +26,58 @@ import aisuite as ai
from ..web.guard import check_url
def _find_tools_dir() -> Path:
"""查找 tools 目录（从当前模块路径向上找，或用环境变量）。"""
# 1. 环境变量优先
env_path = os.environ.get("OPENMESH_TOOLS_DIR", "")
if env_path and Path(env_path).exists():
return Path(env_path)
# 2. 从模块路径向上找（打包后 coworker 在 _internal/ 里，tools 在上层）
try:
here = Path(__file__).resolve().parent
for p in [here, here.parent, here.parent.parent]:
candidate = p / "tools"
if candidate.exists():
return candidate
except Exception:
pass
return Path("tools")
def _find_worker_script(tools_dir: Path) -> Path:
"""查找 browser_worker.py 脚本。
查找顺序：
1. 环境变量 BROWSER_WORKER_PATH
2. tools/python/browser_worker.py（运行时预放置的位置）
3. 当前模块同目录（开发环境，PYZ 未打包时）
"""
# 1. 环境变量
env_path = os.environ.get("BROWSER_WORKER_PATH", "")
if env_path and Path(env_path).exists():
return Path(env_path)
# 2. tools/python/browser_worker.py（打包后放这里最稳）
candidate = tools_dir / "python" / "browser_worker.py"
if candidate.exists():
return candidate
# 3. 当前模块同目录
here = Path(__file__).resolve().parent
candidate = here / "browser_worker.py"
if candidate.exists():
return candidate
# 4. 兜底返回同目录路径（报错时能看到在哪找的）
return here / "browser_worker.py"
_TOOLS_DIR = _find_tools_dir()
_WORKER_SCRIPT = _find_worker_script(_TOOLS_DIR)
def _meta(
name: str, *, approval: bool = False, capabilities: Optional[list[str]] = None
):
@@ -62,16 +120,21 @@ def _attach(fn: Callable[..., Any], schema: dict[str, Any], *, approval: bool =
class _BrowserController:
"""浏览器控制器 — 通过子进程 worker 运行 Playwright。
为什么用子进程：
后端是 PyInstaller 打包的 Python，playwright 的 C 扩展在里面会崩；
tools/python/ 是原生 Python，playwright + .venv 里的包能正常运行。
"""
def __init__(self) -> None:
self._lock = threading.RLock()
self._playwright = None
self._browser = None
self._context = None
self._page = None
self._error: Optional[str] = None
self._executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="coworker-browser"
)
self._proc: Optional[subprocess.Popen] = None
self._request_id = 0
self._error: Optional[str] = None
self._state: dict[str, Any] = {
"open": False,
"url": "",
@@ -89,137 +152,182 @@ class _BrowserController:
self._state.update(changes)
self._state["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def _refresh_page_state(self) -> None:
if self._page is None:
self._touch(open=False, status="closed", url="", title="", controls=[])
return
def _ensure_worker(self) -> Optional[dict[str, Any]]:
"""确保 worker 子进程已启动。返回 None 表示成功，返回 dict 表示错误。"""
if self._proc is not None and self._proc.poll() is None:
return None
self._error = None
try:
snap = _snapshot(self._page, 2000)
self._touch(
open=True,
status="open",
url=self._page.url,
title=self._page.title(),
controls=snap.get("controls", [])[:30],
python_exe = _TOOLS_DIR / "python" / "python.exe"
if not python_exe.exists():
return {
"error": "Browser automation requires tools/python/python.exe (not found).",
"details": f"Expected at: {python_exe}",
}
env = os.environ.copy()
env["OPENMESH_TOOLS_DIR"] = str(_TOOLS_DIR)
env["PYTHONIOENCODING"] = "utf-8"
env["PYTHONUNBUFFERED"] = "1"
self._proc = subprocess.Popen(
[str(python_exe), str(_WORKER_SCRIPT)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
)
except Exception as exc:
self._touch(open=True, status="error", last_error=str(exc))
def _setup_error(self, exc: Exception) -> dict[str, str]:
return {
"error": (
"Interactive browser automation requires Playwright. Install it with "
"`pip install playwright` and `python -m playwright install chromium`."
),
"details": str(exc),
}
def page(self):
with self._lock:
if self._error:
return None, {"error": self._error}
if self._page is not None:
return self._page, None
# 等待 ready 信号
ready_line = self._proc.stdout.readline()
if not ready_line:
stderr = self._proc.stderr.read().decode("utf-8", errors="replace")
return {
"error": "Browser worker failed to start.",
"details": stderr[:500],
}
try:
from playwright.sync_api import sync_playwright
ready = json.loads(ready_line.decode("utf-8").strip())
if ready.get("status") != "ready":
return {
"error": "Browser worker did not report ready.",
"details": str(ready),
}
except json.JSONDecodeError:
return {
"error": "Browser worker sent invalid ready signal.",
"details": ready_line.decode("utf-8", errors="replace")[:200],
}
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(headless=False)
self._context = self._browser.new_context(
viewport={"width": 1280, "height": 900}
)
self._page = self._context.new_page()
self._touch(
open=True, status="open", last_action="open browser", last_error=""
)
return self._page, None
return None
except Exception as exc:
self._error = str(exc)
return {
"error": (
"Interactive browser automation requires Playwright. "
"Make sure tools/python/ and tools/playwright/chromium-1234/ are present."
),
"details": str(exc),
}
def _send_command(self, action: str, params: Optional[dict] = None) -> dict[str, Any]:
"""向 worker 发送一条命令并等待响应。"""
with self._lock:
err = self._ensure_worker()
if err:
return err
self._request_id += 1
req_id = self._request_id
req = json.dumps(
{"id": req_id, "action": action, "params": params or {}},
ensure_ascii=False,
) + "\n"
try:
self._proc.stdin.write(req.encode("utf-8"))
self._proc.stdin.flush()
except Exception as exc:
self._touch(open=False, status="error", last_error=str(exc))
return None, self._setup_error(exc)
self._kill_worker()
return {"error": f"failed to send to browser worker: {exc}"}
try:
line = self._proc.stdout.readline()
if not line:
stderr = self._proc.stderr.read().decode("utf-8", errors="replace")
self._kill_worker()
return {
"error": "browser worker exited unexpectedly",
"details": stderr[:500],
}
resp = json.loads(line.decode("utf-8").strip())
if resp.get("id") != req_id:
return {"error": f"response id mismatch: {resp.get('id')} vs {req_id}"}
if "error" in resp:
self._touch(last_action=action, last_result="error", last_error=resp["error"])
return {"error": resp["error"]}
self._touch(last_action=action, last_result="ok", last_error="")
# 同步状态
if "url" in resp:
self._state["url"] = resp.get("url", "")
if "title" in resp:
self._state["title"] = resp.get("title", "")
if "controls" in resp:
self._state["controls"] = resp.get("controls", [])
if "screenshot_data_url" in resp:
self._state["screenshot_data_url"] = resp["screenshot_data_url"]
if action == "close":
self._state["open"] = False
self._state["status"] = "closed"
self._kill_worker()
elif action in ("open_url", "read_page", "click", "type", "select", "wait"):
self._state["open"] = True
self._state["status"] = "open"
return resp
except Exception as exc:
self._kill_worker()
return {"error": f"failed to read browser worker response: {exc}"}
def _kill_worker(self) -> None:
try:
if self._proc and self._proc.poll() is None:
self._proc.terminate()
self._proc.wait(timeout=3)
except Exception:
try:
if self._proc and self._proc.poll() is None:
self._proc.kill()
except Exception:
pass
finally:
self._proc = None
def _submit(self, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
return self._executor.submit(fn).result()
def close(self) -> dict[str, Any]:
return self._submit(self._close_locked)
def _close_locked(self) -> dict[str, Any]:
with self._lock:
try:
if self._context is not None:
self._context.close()
if self._browser is not None:
self._browser.close()
if self._playwright is not None:
self._playwright.stop()
except Exception as exc:
return {"error": str(exc)}
finally:
self._playwright = None
self._browser = None
self._context = None
self._page = None
self._touch(open=False, status="closed", url="", title="", controls=[])
return {"ok": True}
def _do():
with self._lock:
if self._proc is None:
return {"ok": True}
return self._send_command("close")
return self._submit(_do)
def state(self) -> dict[str, Any]:
return self._submit(self._state_locked)
def _state_locked(self) -> dict[str, Any]:
with self._lock:
self._refresh_page_state()
return dict(self._state)
def _do():
with self._lock:
if self._proc is None:
return dict(self._state)
resp = self._send_command("state")
if "error" in resp:
return dict(self._state)
return {**dict(self._state), **{k: v for k, v in resp.items() if k != "id"}}
return self._submit(_do)
def screenshot(self) -> dict[str, Any]:
return self._submit(self._screenshot_locked)
def _screenshot_locked(self) -> dict[str, Any]:
with self._lock:
page, err = self.page()
if err:
return err
try:
png = page.screenshot(full_page=False)
data_url = "data:image/png;base64," + base64.b64encode(png).decode(
"ascii"
)
self._touch(
screenshot_data_url=data_url,
last_action="screenshot",
last_result="ok",
last_error="",
)
self._refresh_page_state()
def _do():
with self._lock:
resp = self._send_command("screenshot")
if "error" in resp:
return resp
return {"ok": True, **dict(self._state)}
except Exception as exc:
self._touch(
last_action="screenshot", last_result="error", last_error=str(exc)
)
return {"error": str(exc)}
return self._submit(_do)
def call(self, action: str, fn: Callable[[Any], dict[str, Any]]) -> dict[str, Any]:
def run() -> dict[str, Any]:
with self._lock:
page, err = self.page()
if err:
return err
self._touch(last_action=action, last_result="running", last_error="")
try:
out = fn(page)
except Exception as exc:
out = {"error": str(exc)}
if "error" in out:
self._touch(
last_action=action,
last_result="error",
last_error=str(out["error"]),
)
else:
self._refresh_page_state()
self._touch(last_action=action, last_result="ok", last_error="")
return out
"""兼容旧的 call 接口 — 直接通过 worker 执行 action。
return self._submit(run)
注意：fn 参数被忽略，所有逻辑在 worker 里实现。
action 名直接映射到 worker 的 action 名。
"""
def _do():
with self._lock:
# 从 fn 的闭包或通过 action 名构造 params
# 为了兼容旧代码，这里用 action 名直接发命令
# 参数通过额外机制传递（见下方每个工具函数的实现）
return self._send_command(action)
return self._submit(_do)
_BROWSER = _BrowserController()
@@ -237,110 +345,25 @@ def browser_close_session() -> dict[str, Any]:
return _BROWSER.close()
def _cap(value: int, default: int = 20000, upper: int = 100000) -> int:
try:
return max(1, min(int(value or default), upper))
except Exception:
return default
_BROWSER = _BrowserController()
def _target_locator(page, target: str):
target = target.strip()
if target.startswith("text="):
return page.get_by_text(target[5:], exact=False).first
if target.startswith("role="):
role_name = target[5:]
role, _, name = role_name.partition(":")
return page.get_by_role(role.strip(), name=name.strip() or None).first
try:
return page.locator(target).first
except Exception:
return page.get_by_text(target, exact=False).first
def browser_state() -> dict[str, Any]:
return _BROWSER.state()
def _safe_call(fn: Callable[[], Any]) -> dict[str, Any]:
try:
return fn()
except Exception as exc:
return {"error": str(exc)}
def browser_take_screenshot() -> dict[str, Any]:
return _BROWSER.screenshot()
def _browser_call(action: str, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
return _BROWSER.call(action, lambda _page: fn())
def browser_close_session() -> dict[str, Any]:
return _BROWSER.close()
_SNAPSHOT_JS = """
() => {
const visible = (el) => {
const style = window.getComputedStyle(el);
const rect = el.getBoundingClientRect();
return style && style.visibility !== 'hidden' && style.display !== 'none' && rect.width > 0 && rect.height > 0;
};
const labelFor = (el) => {
if (el.labels && el.labels.length) return Array.from(el.labels).map(l => l.innerText.trim()).filter(Boolean).join(' ');
const id = el.getAttribute('id');
if (id) {
const label = document.querySelector(`label[for="${CSS.escape(id)}"]`);
if (label) return label.innerText.trim();
}
return '';
};
const describe = (el, i) => ({
index: i,
tag: el.tagName.toLowerCase(),
type: el.getAttribute('type') || '',
id: el.getAttribute('id') || '',
name: el.getAttribute('name') || '',
role: el.getAttribute('role') || '',
aria: el.getAttribute('aria-label') || '',
label: labelFor(el),
placeholder: el.getAttribute('placeholder') || '',
text: (el.innerText || el.value || '').trim().slice(0, 200),
href: el.getAttribute('href') || '',
selectorHint: el.getAttribute('id') ? `#${CSS.escape(el.getAttribute('id'))}` : (el.getAttribute('name') ? `[name="${el.getAttribute('name')}"]` : '')
});
const controls = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[contenteditable="true"]'))
.filter(visible)
.slice(0, 120)
.map(describe);
return {
title: document.title,
url: location.href,
text: document.body ? document.body.innerText : '',
controls
};
}
"""
def _snapshot(page, max_chars: int) -> dict[str, Any]:
data = page.evaluate(_SNAPSHOT_JS)
text = re.sub(r"\n{3,}", "\n\n", str(data.get("text") or ""))
cap = _cap(max_chars)
return {
"title": data.get("title"),
"url": data.get("url"),
"text": text[:cap],
"truncated": len(text) > cap,
"controls": data.get("controls") or [],
}
def redirect_refusal(requested: str, final: str) -> Optional[str]:
"""A refusal reason if navigation LANDED somewhere the address guard would refuse.
`check_url` vets the URL the model supplied; Playwright then follows redirects, and the
hop that actually loads is a different address the guard never saw (OPE-124). A public
shortener can land on the cloud metadata endpoint or a router admin page, and the
approval the user gave was for the first URL, not this one.
The request has already gone out by the time this runs — it cannot be prevented here.
What it prevents is the agent READING the page or interacting with it. Later
JavaScript- or meta-refresh-driven navigation is still unchecked; only a proxy that
vets every hop closes that, which is the larger design this defers."""
if not final or final == requested:
return None
return check_url(final)
def _worker_call(action: str, params: Optional[dict] = None) -> dict[str, Any]:
"""通过 worker 子进程执行浏览器操作。"""
# 用 _send_command，但它是私有方法——通过 _BROWSER 的内部机制调用
return _BROWSER._send_command(action, params)
def make_browser_automation_tools(
@@ -349,13 +372,7 @@ def make_browser_automation_tools(
tools: list[Callable[..., Any]] = []
def _readable_source(raw: str) -> tuple[Any, dict[str, Any] | None]:
"""A local file to upload, resolved inside a granted root (OPE-122).
These tools touch the filesystem but classify EXTERNAL, so the permission engine's
root scoping — which only runs for WRITE_LOCAL — never sees them. Without this
check the only thing between `~/.ssh/id_rsa` and a web form is someone reading the
approval card. Mirrors `email_send`'s attachment rule, which solves the same
problem for outgoing mail."""
"""A local file to upload, resolved inside a granted root (OPE-122)."""
allowed = [r.path for r in (roots or [])]
if not allowed:
return None, {"error": "no session directory is available to upload from"}
@@ -365,8 +382,7 @@ def make_browser_automation_tools(
return path, None
def _writable_target(raw: str) -> tuple[Any, dict[str, Any] | None]:
"""Where a screenshot may land: inside a WRITABLE granted root. An unnamed target
keeps the temp-file default, which is not a place the user asked us to protect."""
"""Where a screenshot may land: inside a WRITABLE granted root."""
writable = [r.path for r in (roots or []) if r.writable]
if not writable:
return None, {"error": "no writable session directory for the screenshot"}
@@ -382,25 +398,11 @@ def make_browser_automation_tools(
) -> dict[str, Any]:
if not url.lower().startswith(("http://", "https://")):
return {"error": "url must start with http:// or https://"}
# Same address guard as web_fetch. This is approval gated, so it is defense in
# depth, not the primary control. It checks the initial model supplied URL only;
# redirects that the browser follows internally are not hop checked here.
# URL guard (same as web_fetch — defense in depth)
blocked = check_url(url)
if blocked:
return {"error": blocked}
def _open(page):
page.goto(url, wait_until=wait_until, timeout=30000)
landed = redirect_refusal(url, page.url)
if landed:
# Leave nothing readable behind: the next snapshot/get_text must not be
# able to lift content off a page we just refused.
final = page.url
page.goto("about:blank")
return {"error": f"redirected to {final} — {landed}"}
return {"ok": True, "url": page.url}
return _BROWSER.call("open_url", _open)
return _worker_call("open_url", {"url": url, "wait_until": wait_until})
browser_open_url.__name__ = "browser_open_url"
tools.append(
@@ -417,7 +419,7 @@ def make_browser_automation_tools(
)
def browser_read_page(max_chars: int = 20000) -> dict[str, Any]:
return _BROWSER.call("snapshot", lambda page: _snapshot(page, max_chars))
return _worker_call("read_page", {"max_chars": max_chars})
browser_read_page.__name__ = "browser_read_page"
tools.append(
@@ -436,13 +438,7 @@ def make_browser_automation_tools(
)
def browser_click(target: str) -> dict[str, Any]:
return _BROWSER.call(
"click",
lambda page: (
_target_locator(page, target).click(timeout=10000),
{"ok": True, "url": page.url},
)[1],
)
return _worker_call("click", {"target": target})
browser_click.__name__ = "browser_click"
tools.append(
@@ -459,15 +455,7 @@ def make_browser_automation_tools(
)
def browser_type(target: str, text: str, clear: bool = True) -> dict[str, Any]:
def run(page):
loc = _target_locator(page, target)
if clear:
loc.fill(text, timeout=10000)
else:
loc.type(text, timeout=10000)
return {"ok": True, "url": page.url}
return _BROWSER.call("type", run)
return _worker_call("type", {"target": target, "text": text, "clear": clear})
browser_type.__name__ = "browser_type"
tools.append(
@@ -488,13 +476,7 @@ def make_browser_automation_tools(
)
def browser_select(target: str, value: str) -> dict[str, Any]:
return _BROWSER.call(
"select",
lambda page: (
_target_locator(page, target).select_option(value, timeout=10000),
{"ok": True, "url": page.url},
)[1],
)
return _worker_call("select", {"target": target, "value": value})
browser_select.__name__ = "browser_select"
tools.append(
@@ -516,15 +498,7 @@ def make_browser_automation_tools(
return err
if not file_path.exists():
return {"error": f"file not found: {file_path}"}
return _BROWSER.call(
"upload_file",
lambda page: (
_target_locator(page, target).set_input_files(
str(file_path), timeout=10000
),
{"ok": True, "path": str(file_path)},
)[1],
)
return _worker_call("upload_file", {"target": target, "path": str(file_path)})
browser_upload_file.__name__ = "browser_upload_file"
tools.append(
@@ -541,16 +515,9 @@ def make_browser_automation_tools(
)
def browser_wait(milliseconds: int = 1000, target: str = "") -> dict[str, Any]:
def run(page):
if target:
_target_locator(page, target).wait_for(
timeout=max(1, int(milliseconds or 1000))
)
else:
page.wait_for_timeout(max(1, min(int(milliseconds or 1000), 30000)))
return {"ok": True, "url": page.url}
return _BROWSER.call("wait", run)
return _worker_call(
"wait", {"milliseconds": milliseconds, "target": target}
)
browser_wait.__name__ = "browser_wait"
tools.append(
@@ -567,24 +534,13 @@ def make_browser_automation_tools(
)
def browser_screenshot(path: str = "") -> dict[str, Any]:
params = {}
if path:
_target, target_err = _writable_target(path)
if target_err:
return target_err
def run(page):
out = (
_target
if path
else (
Path(tempfile.gettempdir()) / "coworker-browser-screenshot.png"
).resolve()
)
out.parent.mkdir(parents=True, exist_ok=True)
page.screenshot(path=str(out), full_page=True)
return {"ok": True, "path": str(out), "url": page.url}
return _BROWSER.call("screenshot", run)
params["path"] = str(_target)
return _worker_call("screenshot", params)
browser_screenshot.__name__ = "browser_screenshot"
tools.append(