feat: OpenMesh 基础平台与 MD/PDF 转换技能

- 后端: coworker 智能体框架, WS API, 文件上传, 附件处理
- 前端: Open WebUI, 文件全量走 upload API (含 MD/TXT/JSON 等文本类)
- 技能: md-to-office (pandoc + wkhtmltopdf)
- 修复: 上传文件路径丢失, Agent 搜索浪费, 输出文件跑到 uploads/
- 打包: PyInstaller one-dir, 预打包 pandoc/wkhtmltopdf/chromium

coworker/connectors/browser_automation.py

@@ -0,0 +1,620 @@
+"""Playwright-backed browser automation tools for Cowork.
+
+The dependency is optional. If Playwright or its browser binaries are not installed, the
+tools return a clear setup error instead of breaking engine construction.
+"""
+
+from __future__ import annotations
+
+import re
+import tempfile
+import threading
+import time
+import base64
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+import aisuite as ai
+
+from ..web.guard import check_url
+
+
+def _meta(
+    name: str, *, approval: bool = False, capabilities: Optional[list[str]] = None
+):
+    return ai.ToolMetadata(
+        name=name,
+        category="connector",
+        risk_level="medium" if approval else "low",
+        capabilities=capabilities or ["browser"],
+        requires_approval=approval,
+    )
+
+
+def _schema(
+    name: str, description: str, properties: dict[str, Any], required: list[str]
+) -> dict[str, Any]:
+    return {
+        "type": "function",
+        "function": {
+            "name": name,
+            "description": description,
+            "parameters": {
+                "type": "object",
+                "properties": properties,
+                "required": required,
+            },
+        },
+    }
+
+
+def _attach(fn: Callable[..., Any], schema: dict[str, Any], *, approval: bool = True):
+    from .tool_defs import approval_for_tool
+
+    name = schema["function"]["name"]
+    # §36: the tool registry's read/write kind wins for registered tools — reads never gate.
+    approval = approval_for_tool(name, default=approval)
+    fn.__coworker_schema__ = schema
+    fn.__aisuite_tool_metadata__ = _meta(name, approval=approval)
+    fn.__doc__ = schema["function"]["description"]
+    return fn
+
+
+class _BrowserController:
+    def __init__(self) -> None:
+        self._lock = threading.RLock()
+        self._playwright = None
+        self._browser = None
+        self._context = None
+        self._page = None
+        self._error: Optional[str] = None
+        self._executor = ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="coworker-browser"
+        )
+        self._state: dict[str, Any] = {
+            "open": False,
+            "url": "",
+            "title": "",
+            "status": "closed",
+            "last_action": "",
+            "last_result": "",
+            "last_error": "",
+            "screenshot_data_url": "",
+            "updated_at": None,
+            "controls": [],
+        }
+
+    def _touch(self, **changes: Any) -> None:
+        self._state.update(changes)
+        self._state["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+    def _refresh_page_state(self) -> None:
+        if self._page is None:
+            self._touch(open=False, status="closed", url="", title="", controls=[])
+            return
+        try:
+            snap = _snapshot(self._page, 2000)
+            self._touch(
+                open=True,
+                status="open",
+                url=self._page.url,
+                title=self._page.title(),
+                controls=snap.get("controls", [])[:30],
+            )
+        except Exception as exc:
+            self._touch(open=True, status="error", last_error=str(exc))
+
+    def _setup_error(self, exc: Exception) -> dict[str, str]:
+        return {
+            "error": (
+                "Interactive browser automation requires Playwright. Install it with "
+                "`pip install playwright` and `python -m playwright install chromium`."
+            ),
+            "details": str(exc),
+        }
+
+    def page(self):
+        with self._lock:
+            if self._error:
+                return None, {"error": self._error}
+            if self._page is not None:
+                return self._page, None
+            try:
+                from playwright.sync_api import sync_playwright
+
+                self._playwright = sync_playwright().start()
+                self._browser = self._playwright.chromium.launch(headless=False)
+                self._context = self._browser.new_context(
+                    viewport={"width": 1280, "height": 900}
+                )
+                self._page = self._context.new_page()
+                self._touch(
+                    open=True, status="open", last_action="open browser", last_error=""
+                )
+                return self._page, None
+            except Exception as exc:
+                self._touch(open=False, status="error", last_error=str(exc))
+                return None, self._setup_error(exc)
+
+    def _submit(self, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
+        return self._executor.submit(fn).result()
+
+    def close(self) -> dict[str, Any]:
+        return self._submit(self._close_locked)
+
+    def _close_locked(self) -> dict[str, Any]:
+        with self._lock:
+            try:
+                if self._context is not None:
+                    self._context.close()
+                if self._browser is not None:
+                    self._browser.close()
+                if self._playwright is not None:
+                    self._playwright.stop()
+            except Exception as exc:
+                return {"error": str(exc)}
+            finally:
+                self._playwright = None
+                self._browser = None
+                self._context = None
+                self._page = None
+                self._touch(open=False, status="closed", url="", title="", controls=[])
+            return {"ok": True}
+
+    def state(self) -> dict[str, Any]:
+        return self._submit(self._state_locked)
+
+    def _state_locked(self) -> dict[str, Any]:
+        with self._lock:
+            self._refresh_page_state()
+            return dict(self._state)
+
+    def screenshot(self) -> dict[str, Any]:
+        return self._submit(self._screenshot_locked)
+
+    def _screenshot_locked(self) -> dict[str, Any]:
+        with self._lock:
+            page, err = self.page()
+            if err:
+                return err
+            try:
+                png = page.screenshot(full_page=False)
+                data_url = "data:image/png;base64," + base64.b64encode(png).decode(
+                    "ascii"
+                )
+                self._touch(
+                    screenshot_data_url=data_url,
+                    last_action="screenshot",
+                    last_result="ok",
+                    last_error="",
+                )
+                self._refresh_page_state()
+                return {"ok": True, **dict(self._state)}
+            except Exception as exc:
+                self._touch(
+                    last_action="screenshot", last_result="error", last_error=str(exc)
+                )
+                return {"error": str(exc)}
+
+    def call(self, action: str, fn: Callable[[Any], dict[str, Any]]) -> dict[str, Any]:
+        def run() -> dict[str, Any]:
+            with self._lock:
+                page, err = self.page()
+                if err:
+                    return err
+                self._touch(last_action=action, last_result="running", last_error="")
+                try:
+                    out = fn(page)
+                except Exception as exc:
+                    out = {"error": str(exc)}
+                if "error" in out:
+                    self._touch(
+                        last_action=action,
+                        last_result="error",
+                        last_error=str(out["error"]),
+                    )
+                else:
+                    self._refresh_page_state()
+                    self._touch(last_action=action, last_result="ok", last_error="")
+                return out
+
+        return self._submit(run)
+
+
+_BROWSER = _BrowserController()
+
+
+def browser_state() -> dict[str, Any]:
+    return _BROWSER.state()
+
+
+def browser_take_screenshot() -> dict[str, Any]:
+    return _BROWSER.screenshot()
+
+
+def browser_close_session() -> dict[str, Any]:
+    return _BROWSER.close()
+
+
+def _cap(value: int, default: int = 20000, upper: int = 100000) -> int:
+    try:
+        return max(1, min(int(value or default), upper))
+    except Exception:
+        return default
+
+
+def _target_locator(page, target: str):
+    target = target.strip()
+    if target.startswith("text="):
+        return page.get_by_text(target[5:], exact=False).first
+    if target.startswith("role="):
+        role_name = target[5:]
+        role, _, name = role_name.partition(":")
+        return page.get_by_role(role.strip(), name=name.strip() or None).first
+    try:
+        return page.locator(target).first
+    except Exception:
+        return page.get_by_text(target, exact=False).first
+
+
+def _safe_call(fn: Callable[[], Any]) -> dict[str, Any]:
+    try:
+        return fn()
+    except Exception as exc:
+        return {"error": str(exc)}
+
+
+def _browser_call(action: str, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
+    return _BROWSER.call(action, lambda _page: fn())
+
+
+_SNAPSHOT_JS = """
+() => {
+  const visible = (el) => {
+    const style = window.getComputedStyle(el);
+    const rect = el.getBoundingClientRect();
+    return style && style.visibility !== 'hidden' && style.display !== 'none' && rect.width > 0 && rect.height > 0;
+  };
+  const labelFor = (el) => {
+    if (el.labels && el.labels.length) return Array.from(el.labels).map(l => l.innerText.trim()).filter(Boolean).join(' ');
+    const id = el.getAttribute('id');
+    if (id) {
+      const label = document.querySelector(`label[for="${CSS.escape(id)}"]`);
+      if (label) return label.innerText.trim();
+    }
+    return '';
+  };
+  const describe = (el, i) => ({
+    index: i,
+    tag: el.tagName.toLowerCase(),
+    type: el.getAttribute('type') || '',
+    id: el.getAttribute('id') || '',
+    name: el.getAttribute('name') || '',
+    role: el.getAttribute('role') || '',
+    aria: el.getAttribute('aria-label') || '',
+    label: labelFor(el),
+    placeholder: el.getAttribute('placeholder') || '',
+    text: (el.innerText || el.value || '').trim().slice(0, 200),
+    href: el.getAttribute('href') || '',
+    selectorHint: el.getAttribute('id') ? `#${CSS.escape(el.getAttribute('id'))}` : (el.getAttribute('name') ? `[name="${el.getAttribute('name')}"]` : '')
+  });
+  const controls = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[contenteditable="true"]'))
+    .filter(visible)
+    .slice(0, 120)
+    .map(describe);
+  return {
+    title: document.title,
+    url: location.href,
+    text: document.body ? document.body.innerText : '',
+    controls
+  };
+}
+"""
+
+
+def _snapshot(page, max_chars: int) -> dict[str, Any]:
+    data = page.evaluate(_SNAPSHOT_JS)
+    text = re.sub(r"\n{3,}", "\n\n", str(data.get("text") or ""))
+    cap = _cap(max_chars)
+    return {
+        "title": data.get("title"),
+        "url": data.get("url"),
+        "text": text[:cap],
+        "truncated": len(text) > cap,
+        "controls": data.get("controls") or [],
+    }
+
+
+def redirect_refusal(requested: str, final: str) -> Optional[str]:
+    """A refusal reason if navigation LANDED somewhere the address guard would refuse.
+
+    `check_url` vets the URL the model supplied; Playwright then follows redirects, and the
+    hop that actually loads is a different address the guard never saw (OPE-124). A public
+    shortener can land on the cloud metadata endpoint or a router admin page, and the
+    approval the user gave was for the first URL, not this one.
+
+    The request has already gone out by the time this runs — it cannot be prevented here.
+    What it prevents is the agent READING the page or interacting with it. Later
+    JavaScript- or meta-refresh-driven navigation is still unchecked; only a proxy that
+    vets every hop closes that, which is the larger design this defers."""
+    if not final or final == requested:
+        return None
+    return check_url(final)
+
+
+def make_browser_automation_tools(
+    *, roots: Optional[list[Any]] = None
+) -> list[Callable[..., Any]]:
+    tools: list[Callable[..., Any]] = []
+
+    def _readable_source(raw: str) -> tuple[Any, dict[str, Any] | None]:
+        """A local file to upload, resolved inside a granted root (OPE-122).
+
+        These tools touch the filesystem but classify EXTERNAL, so the permission engine's
+        root scoping — which only runs for WRITE_LOCAL — never sees them. Without this
+        check the only thing between `~/.ssh/id_rsa` and a web form is someone reading the
+        approval card. Mirrors `email_send`'s attachment rule, which solves the same
+        problem for outgoing mail."""
+        allowed = [r.path for r in (roots or [])]
+        if not allowed:
+            return None, {"error": "no session directory is available to upload from"}
+        path = Path(str(raw)).expanduser().resolve()
+        if not any(path.is_relative_to(root) for root in allowed):
+            return None, {"error": f"{path} is outside the session's directories"}
+        return path, None
+
+    def _writable_target(raw: str) -> tuple[Any, dict[str, Any] | None]:
+        """Where a screenshot may land: inside a WRITABLE granted root. An unnamed target
+        keeps the temp-file default, which is not a place the user asked us to protect."""
+        writable = [r.path for r in (roots or []) if r.writable]
+        if not writable:
+            return None, {"error": "no writable session directory for the screenshot"}
+        path = Path(str(raw)).expanduser().resolve()
+        if not any(path.is_relative_to(root) for root in writable):
+            return None, {
+                "error": f"{path} is outside the session's writable directories"
+            }
+        return path, None
+
+    def browser_open_url(
+        url: str, wait_until: str = "domcontentloaded"
+    ) -> dict[str, Any]:
+        if not url.lower().startswith(("http://", "https://")):
+            return {"error": "url must start with http:// or https://"}
+        # Same address guard as web_fetch. This is approval gated, so it is defense in
+        # depth, not the primary control. It checks the initial model supplied URL only;
+        # redirects that the browser follows internally are not hop checked here.
+        blocked = check_url(url)
+        if blocked:
+            return {"error": blocked}
+
+        def _open(page):
+            page.goto(url, wait_until=wait_until, timeout=30000)
+            landed = redirect_refusal(url, page.url)
+            if landed:
+                # Leave nothing readable behind: the next snapshot/get_text must not be
+                # able to lift content off a page we just refused.
+                final = page.url
+                page.goto("about:blank")
+                return {"error": f"redirected to {final} — {landed}"}
+            return {"ok": True, "url": page.url}
+
+        return _BROWSER.call("open_url", _open)
+
+    browser_open_url.__name__ = "browser_open_url"
+    tools.append(
+        _attach(
+            browser_open_url,
+            _schema(
+                "browser_open_url",
+                "Open a URL in the local Playwright browser session.",
+                {"url": {"type": "string"}, "wait_until": {"type": "string"}},
+                ["url"],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_read_page(max_chars: int = 20000) -> dict[str, Any]:
+        return _BROWSER.call("snapshot", lambda page: _snapshot(page, max_chars))
+
+    browser_read_page.__name__ = "browser_read_page"
+    tools.append(
+        _attach(
+            browser_read_page,
+            _schema(
+                "browser_read_page",
+                "Read the current page: its text plus visible controls and selector "
+                "hints (for browser_click/browser_type). Not an image — use "
+                "browser_screenshot for pixels.",
+                {"max_chars": {"type": "integer"}},
+                [],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_click(target: str) -> dict[str, Any]:
+        return _BROWSER.call(
+            "click",
+            lambda page: (
+                _target_locator(page, target).click(timeout=10000),
+                {"ok": True, "url": page.url},
+            )[1],
+        )
+
+    browser_click.__name__ = "browser_click"
+    tools.append(
+        _attach(
+            browser_click,
+            _schema(
+                "browser_click",
+                "Click a visible page element by CSS selector, text=label, role=button:Name, or text fallback. Requires approval.",
+                {"target": {"type": "string"}},
+                ["target"],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_type(target: str, text: str, clear: bool = True) -> dict[str, Any]:
+        def run(page):
+            loc = _target_locator(page, target)
+            if clear:
+                loc.fill(text, timeout=10000)
+            else:
+                loc.type(text, timeout=10000)
+            return {"ok": True, "url": page.url}
+
+        return _BROWSER.call("type", run)
+
+    browser_type.__name__ = "browser_type"
+    tools.append(
+        _attach(
+            browser_type,
+            _schema(
+                "browser_type",
+                "Fill or type into an input, textarea, or editable element. Requires approval.",
+                {
+                    "target": {"type": "string"},
+                    "text": {"type": "string"},
+                    "clear": {"type": "boolean"},
+                },
+                ["target", "text"],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_select(target: str, value: str) -> dict[str, Any]:
+        return _BROWSER.call(
+            "select",
+            lambda page: (
+                _target_locator(page, target).select_option(value, timeout=10000),
+                {"ok": True, "url": page.url},
+            )[1],
+        )
+
+    browser_select.__name__ = "browser_select"
+    tools.append(
+        _attach(
+            browser_select,
+            _schema(
+                "browser_select",
+                "Select an option in a dropdown by selector and option value/label. Requires approval.",
+                {"target": {"type": "string"}, "value": {"type": "string"}},
+                ["target", "value"],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_upload_file(target: str, path: str) -> dict[str, Any]:
+        file_path, err = _readable_source(path)
+        if err:
+            return err
+        if not file_path.exists():
+            return {"error": f"file not found: {file_path}"}
+        return _BROWSER.call(
+            "upload_file",
+            lambda page: (
+                _target_locator(page, target).set_input_files(
+                    str(file_path), timeout=10000
+                ),
+                {"ok": True, "path": str(file_path)},
+            )[1],
+        )
+
+    browser_upload_file.__name__ = "browser_upload_file"
+    tools.append(
+        _attach(
+            browser_upload_file,
+            _schema(
+                "browser_upload_file",
+                "Upload a local file through a file input. Requires approval.",
+                {"target": {"type": "string"}, "path": {"type": "string"}},
+                ["target", "path"],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_wait(milliseconds: int = 1000, target: str = "") -> dict[str, Any]:
+        def run(page):
+            if target:
+                _target_locator(page, target).wait_for(
+                    timeout=max(1, int(milliseconds or 1000))
+                )
+            else:
+                page.wait_for_timeout(max(1, min(int(milliseconds or 1000), 30000)))
+            return {"ok": True, "url": page.url}
+
+        return _BROWSER.call("wait", run)
+
+    browser_wait.__name__ = "browser_wait"
+    tools.append(
+        _attach(
+            browser_wait,
+            _schema(
+                "browser_wait",
+                "Wait for a duration or for a target element to appear.",
+                {"milliseconds": {"type": "integer"}, "target": {"type": "string"}},
+                [],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_screenshot(path: str = "") -> dict[str, Any]:
+        if path:
+            _target, target_err = _writable_target(path)
+            if target_err:
+                return target_err
+
+        def run(page):
+            out = (
+                _target
+                if path
+                else (
+                    Path(tempfile.gettempdir()) / "coworker-browser-screenshot.png"
+                ).resolve()
+            )
+            out.parent.mkdir(parents=True, exist_ok=True)
+            page.screenshot(path=str(out), full_page=True)
+            return {"ok": True, "path": str(out), "url": page.url}
+
+        return _BROWSER.call("screenshot", run)
+
+    browser_screenshot.__name__ = "browser_screenshot"
+    tools.append(
+        _attach(
+            browser_screenshot,
+            _schema(
+                "browser_screenshot",
+                "Save a full-page screenshot of the current browser page and return the local path.",
+                {"path": {"type": "string"}},
+                [],
+            ),
+            approval=True,
+        )
+    )
+
+    def browser_close() -> dict[str, Any]:
+        return browser_close_session()
+
+    browser_close.__name__ = "browser_close"
+    tools.append(
+        _attach(
+            browser_close,
+            _schema(
+                "browser_close",
+                "Close the local Playwright browser session.",
+                {},
+                [],
+            ),
+            approval=True,
+        )
+    )
+
+    return tools