/** * RCA (Root Cause Analysis) runner — v2 AI-driven. * * Two-phase design (no AI SDK tool-loop required): * * Phase 1 — Plan: * AI receives alert + context + system prompt (diagnosis workflow). * Outputs a JSON plan: which diagnosis commands to run on which device. * * Execute — We run the plan: * For each command → aiCapability("netcatty/inspectionRunCommand", { deviceId, command }) * Collect stdout into evidence bundle. * * Phase 2 — Synthesize: * AI receives alert + metrics + ALL command outputs. * Outputs final RCAResult JSON (root causes + recommendations). * * Deterministic fallback — if any phase fails, we use local heuristics * (inspection.runCommand still runs real SSH commands). */ import type { AlertRecord } from "../../inspection/inspectionApi"; import { STORAGE_KEY_HOSTS, STORAGE_KEY_KEYS, } from "../../config/storageKeys"; import { decryptHosts, decryptKeys, } from "../../persistence/secureFieldAdapter"; export interface RunRCAOptions { forceDeterministic?: boolean; onTool?: (name: string, ok: boolean, summary: string) => void; } export interface RCAResult { alertId: string; deviceId: string; deviceName: string; confidence: number; rootCauses: string[]; evidence: Array<{ tool: string; summary: string; keyFindings: string[]; }>; recommendations: Array<{ priority: "immediate" | "short_term" | "long_term"; action: string; command?: string; risk: "low" | "medium" | "high"; note?: string; }>; summary: string; } export interface RunRCAResult { success: boolean; rca?: RCAResult; rawText?: string; source: "ai" | "fallback"; error?: string; durationMs: number; toolInvocations: Array<{ tool: string; ok: boolean; durationMs: number }>; } // ---------- Bridge (renderer preload) ---------- type Bridge = { aiCapability?: ( rpcMethod: string, params: Record, chatSessionId?: string, ) => Promise; aiFetch?: ( url: string, method: string, headers: Record, body?: string, providerId?: string, skipHostCheck?: boolean, followRedirects?: boolean, skipTLSVerify?: boolean, timeoutMs?: number, ) => Promise<{ ok: boolean; status: number; data: string; error?: string }>; aiAllowlistAddHost?: (baseURL: string) => Promise; aiSyncProviders?: (providers: unknown[]) => Promise; inspection?: { runSingleCommand?: (payload: { hostConfig: HostConfig; command: string; timeoutMs?: number; }) => Promise; }; }; function bridge(): Bridge { return (window as unknown as { netcatty?: Bridge }).netcatty ?? ({} as Bridge); } async function rpc(method: string, params: Record): Promise { const b = bridge(); if (!b.aiCapability) throw new Error("aiCapability bridge unavailable"); const result = await b.aiCapability(method, params); // aiCapability returns handler result directly — if ok: false, surface error if (result && typeof result === "object" && (result as any).ok === false) { throw new Error((result as any).error || "RPC failed"); } return result as T; } async function safeRpc(method: string, params: Record): Promise { try { return await rpc(method, params); } catch (_) { return null; } } // ---------- AI chat completions via aiFetch ---------- interface ProviderInfo { providerId: string; apiKey: string; baseUrl: string; model: string; style?: string; } // Storage keys (keep in sync with infrastructure/config/storageKeys.ts) const AI_PROVIDERS_KEY = "netcatty_ai_providers_v1"; const AI_ACTIVE_PROVIDER_KEY = "netcatty_ai_active_provider_v1"; async function resolveProvider(): Promise { let providers: any[] | null = null; let activeId: string | null = null; try { const raw = localStorage.getItem(AI_PROVIDERS_KEY); if (raw) providers = JSON.parse(raw); activeId = localStorage.getItem(AI_ACTIVE_PROVIDER_KEY); } catch (_) {} if (!Array.isArray(providers) || providers.length === 0) return null; // Prefer active provider, fall back to first enabled one let primary: any = null; if (activeId) primary = providers.find((p) => p.id === activeId); if (!primary) primary = providers.find((p) => p.enabled !== false); if (!primary?.apiKey) return null; // 同步 providers 到主进程(主进程解密 + host allowlist 都依赖它) // Catty SDK 正常流程会自动 sync,RCA 跳过了所以这里手动补一次 const b = bridge(); if (b.aiSyncProviders) { try { await b.aiSyncProviders(providers); } catch (_) {} } const baseUrl = primary.baseURL || ""; const style = primary.style || detectStyle(baseUrl || primary.providerId || ""); const path = style === "anthropic" ? "/v1/messages" : style === "google" ? "/v1beta/models" : "/chat/completions"; return { providerId: primary.id || "default", apiKey: primary.apiKey, // 可能是 enc:v1: 加密 placeholder,主进程 injectApiKeyIntoRequest 会解密替换 baseUrl: (baseUrl.replace(/\/$/, "")) + path, model: primary.defaultModel || "gpt-4o-mini", style, }; } /** 清洗厂商登录 banner / URL / 版权噪音,让 AI 只看到命令输出 */ function _stripVendorBanner(text: string): string { if (!text) return text; let out = text; // 1. 删所有 http/https URL out = out.replace(/https?:\/\/[^\s|,,。]+/g, ""); // 2. 删裸域名 www.xxx.com / 管理门户路径 out = out.replace(/\bwww\.[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?/gi, ""); out = out.replace(/\bmanagement\.[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?/gi, ""); // 3. 删残留的 Web 脚本后缀片段(banner 里可能出现裸 .aspx / .do 等) out = out.replace(/\.(aspx|do|cgi|php|jsp|html?|shtml|asp|cfm|jspx|action)(\/[^\s]*)?\b/gi, ""); // 4. 删整行噪音 out = out .split("\n") .filter(line => { const t = line.trim(); if (!t) return true; // SSH 登录 banner if (/Welcome to (Kylin|Ubuntu|CentOS|Red Hat|RedHat|Debian|SUSE|openSUSE|Fedora|AlmaLinux|Rocky Linux)/i.test(t)) return false; if (/^(Huawei|H3C|Cisco|Ruijie|ZTE|锐捷|中兴|华为)/.test(t) && /(Platform|Software|Operating System|Comware|VRP)/i.test(t)) return false; // 管理门户 / 版权 / 警告 if (/^\s*\*?\s*Management\s*[::]/i.test(t)) return false; if (/^\s*\*?\s*(Copyright|Trademark|All Rights Reserved|For Internal Use|Unauthorized)/i.test(t)) return false; if (/^\s*\*{5,}\s*$/.test(t)) return false; // SSH MOTD(登录欢迎信息) if (/^Last login:\s/i.test(t)) return false; if (/^You have (new )?mail/i.test(t)) return false; if (/^No mail$/i.test(t)) return false; if (/^MOTD(\s|$)/i.test(t)) return false; if (/^Message of the Day/i.test(t)) return false; // Kylin / Ubuntu 的欢迎行(如 "Welcome to Kylin V10 SP1..." 已在上面,这里兜底) if (/^\s*\*?\s*(Welcome|欢迎)/i.test(t) && /(Kylin|Ubuntu|CentOS|Linux|操作系统)/i.test(t)) return false; // 麒麟 OS 的软件协议 / 许可提示 if (/^\s*\*?\s*(License|许可|授权|Service Pack|SP\d+)/i.test(t) && /(Kylin|麒麟)/i.test(t)) return false; return true; }) .join("\n"); // 5. 多空行压缩 out = out.replace(/\n{3,}/g, "\n\n").trim(); return out; } function detectStyle(baseUrlOrId: string): string { const lower = baseUrlOrId.toLowerCase(); if (/anthropic|claude/.test(lower)) return "anthropic"; if (/google|gemini/.test(lower)) return "google"; return "openai"; } const CATALOG_LABELS: Record = { linux: "Linux 服务器", linux_server: "Linux 服务器", windows: "Windows 服务器", huawei_h3c_switch: "华为/H3C 交换机", cisco_switch: "Cisco 交换机", huawei_firewall: "华为防火墙", fortinet_firewall: "飞塔防火墙", }; function catalogLabelOf(catalogId?: string): string { if (!catalogId) return "未知(unknown)"; return CATALOG_LABELS[catalogId] || catalogId; } async function aiChatCompletion( provider: ProviderInfo, system: string, messages: Array<{ role: "user" | "assistant"; content: string }>, ): Promise { const b = bridge(); if (!b.aiFetch) throw new Error("aiFetch bridge unavailable"); // 预注册 host 到主进程 allowlist(独立 RCA 路径不走 Catty SDK,没同步过 provider host) // 幂等调用 — host 已存在时只是刷新 TTL if ((b as any).aiAllowlistAddHost) { try { await (b as any).aiAllowlistAddHost(provider.baseUrl); } catch (_) {} } const body = JSON.stringify({ model: provider.model, messages: [ { role: "system", content: system }, ...messages, ], temperature: 0, stream: false, }); const headers: Record = { "Content-Type": "application/json", // 用 placeholder — 主进程 injectApiKeyIntoRequest 会用 providerId 解密注入真实 key Authorization: `Bearer __IPC_SECURED__`, }; if (provider.style === "anthropic") { headers["anthropic-version"] = "2023-06-01"; } // RCA 分析也是多轮思考 + 大输出,给 120s timeout const resp = await b.aiFetch( provider.baseUrl, "POST", headers, body, provider.providerId, /*skipHostCheck*/ true, /*followRedirects*/ false, /*skipTLSVerify*/ false, /*timeoutMs*/ 1200000, ); if (!resp.ok) throw new Error(`AI ${resp.status}: ${resp.data?.slice(0, 200) || resp.error}`); return parseAiChat(resp.data || "{}", provider.style); } function parseAiChat(data: string, style?: string): string { const parsed = JSON.parse(data || "{}"); if (style === "anthropic") return parsed.content?.[0]?.text ?? ""; if (style === "google") return parsed.candidates?.[0]?.content?.parts?.[0]?.text ?? ""; return parsed.choices?.[0]?.message?.content ?? ""; } // ---------- System prompts ---------- // 内置默认值 —— 存储在 STORAGE_KEY_RCA_PROMPTS 里的自定义 prompt 优先使用 export const DEFAULT_PHASE1_SYSTEM = [ `你是 NetMesh 运维平台的 AI 诊断规划专家。`, `平台会在设备上执行你规划的每条命令,收集 stdout 返回给你。`, `如果设备离线(online=false),不要跑任何需要 SSH 的命令 — 改成先确认网络可达性。`, ``, `## 设备 Catalog 与命令族(必须严格匹配)`, `| catalogId | 设备类型 | Shell | 实际巡检命令(系统已在用) | 推荐诊断命令 |`, `|---|---|---|---|---|`, `| linux | Linux 服务器 | bash/sh | top -bn1 / mpstat 1 1 / free -k / cat /proc/meminfo / df -h | top -bn1 \| head -20, vmstat 1 2, free -m, cat /proc/meminfo \| head -20, df -h, ps aux --sort=-%cpu \| head -10, iostat -x 1 1 2>/dev/null \|\| echo no_iostat, lsof -i, journalctl -xe --no-pager \| tail -50, dmesg \| tail -50 |`, `| windows | Windows 服务器 | PowerShell | Get-WmiObject Win32_Processor / Win32_OperatingSystem / Win32_LogicalDisk / Get-Service | Get-Process \| Sort-Object CPU -Descending \| Select-Object -First 10, Get-CimInstance Win32_OperatingSystem, Get-CimInstance Win32_LogicalDisk, Get-Service \| Where-Object {$_.Status -ne 'Running'}, Get-EventLog -LogName System -Newest 50 |`, `| huawei_h3c_switch | 华为/H3C 交换机 | HUAWEI CLI | display cpu-usage / display memory / display interface brief / display power / display fan | **禁止 display cpu-usage(有刷新动画抓不到完整输出)**;可用命令: display version, display device, display memory, display power, display fan, display logbuffer, display stp brief, display vrrp, display vlan brief, display arp |`, `| cisco_switch | Cisco 交换机 | IOS CLI | 复用华为交换机巡检(暂无专用 adapter) | show version, show processes cpu sorted, show memory summary, show interfaces brief, show interfaces counters errors, show inventory, show log tail 50, show vlan brief, show spanning-tree summary |`, `| huawei_firewall | 华为防火墙 | HUAWEI CLI | 复用华为交换机巡检(暂无专用 adapter) | display version, display cpu-usage, display memory, display firewall session table, display current-configuration, display security-policy count, display server-protect statistics |`, `| fortinet_firewall | 飞塔防火墙 | FortiOS CLI | 复用华为交换机巡检(暂无专用 adapter) | get system status, get system performance status, get system performance cpu, get system performance memory, get firewall session list, get config route static, diagnose debug flow stat |`, ``, `## 输出格式(严格 JSON,不要 markdown 代码块之外的文字)`, `{`, ` "commands": [`, ` { "command": "display cpu-usage", "purpose": "查看设备 CPU 使用率" }`, ` ],`, ` "reasoning": "一句话说明为什么选这些命令"`, `}`, ``, `## 命令规划规则`, `- **命令必须严格从上面"推荐诊断命令"列中选择**,不得臆造未知命令(如 display process、display memory-usage)`, `- **严禁输出任何外部 URL / 厂商官网链接 / 技术文档链接**(如 management.h3c.com、www.kylinos.cn、cisco.com/wiki、forum.huawei 等) —— 你的运行环境在内网,外网链接无法访问,输出它们没有任何意义`, `- **严禁引用厂商文档名称和章节号**(如"参见 H3C S5700 V200R010 配置指南"),设备 firmware / 型号信息必须从诊断命令输出中提取`, `- **严禁推荐访问外网工具**(如 curl/wget 拉远程脚本、apt/yum 从公网仓库安装)`, `- **禁止选择语义重复的命令**(如 display memory 和 display memory-usage 只需选一条)`, `- **禁止选择持续刷新/交互式命令** — display cpu-usage 有动态刷新动画,无法完整抓取输出。请改用 display device(含 CPU 模块状态)或 display power`, `- **关闭分页的前置命令**(让 display logbuffer 完整输出):华为/H3C 用 "screen-length 0 temporary"(用户视图下可用),不要用 "terminal length 0"(那是 system-view 下的)。独立作为 commands 数组里的第一条`, `- **display logbuffer** 建议用 "display logbuffer level warning" 或 "display logbuffer size 512" 缩小输出范围,避免 10000+ 行日志`, `- display version 是只读静态信息,建议放在 plan 里至少一次(用来识别设备型号/固件版本,对根因分析很关键)`, `- CPU 告警 → 至少 2 条命令(display device 看硬件状态 + display logbuffer 查告警)`, `- 内存告警 → 至少 2 条命令(display memory + display logbuffer level warning 查 OOM)`, `- 离线告警 → 只跑连通性命令(ping, traceroute),不要尝试 SSH 命令`, `- 不超过 6 条命令,总耗时不超过 60 秒`, `- 命令必须简短可执行,不含交互式程序、不含 tail -f 等持续输出`, `- 如果 catalogId 是 unknown,从设备名称 / metrics / 告警特征推断,拿不准时先跑 uname -a(Linux)或 display version(华为)或 show version(Cisco)`, ].join("\n"); export const DEFAULT_PHASE2_SYSTEM = [ `你是 NetMesh 智慧运维平台的 AI 根因分析专家。`, `你收到了一条告警 + 最新巡检 metrics + 多条诊断命令的执行输出。`, `请综合这些证据,给出结构化的根因分析和处置建议。`, ``, `## 核心约束 — 内网环境,严格执行`, `- 严禁输出任何外部 URL / 厂商官网链接 / 技术文档链接(如 management.h3c.com、www.kylinos.cn、cisco.com/wiki 等)`, `- 严禁引用厂商文档名称、章节号、版本号(如"H3C S5700 V200R010 配置指南 第 3 章")`, `- 严禁推荐访问外网的工具或步骤(如"从官网下载固件"、"curl 远程脚本"、"apt 公网仓库安装")`, `- 设备固件版本、硬件型号等信息必须从诊断命令的 stdout 输出中提取,不得脑补厂商数据库内容`, `- 所有处置建议必须在**不依赖外网访问**的前提下可执行`, ``, `## 输出格式(严格 JSON,不要 markdown 代码块)`, `{`, ` "confidence": 85,`, ` "rootCauses": ["最可能根因", "次要可能性"],`, ` "summary": "2-3 句摘要,适合运维人员快速阅读",`, ` "recommendations": [`, ` { "priority": "immediate|short_term|long_term", "action": "要做什么", "command": "可选命令", "risk": "low|medium|high", "note": "注意事项" }`, ` ]`, `}`, ``, `## 规则`, `- confidence 基于证据充分性(有 metrics + 命令输出且相互印证 → 80+;只有 metrics → 60;无数据 → 30 以下)`, `- recommendations 至少 2 条,最多 5 条,按紧急程度排序`, `- command 只放确定安全的命令,破坏性操作只在 action 里描述`, `- priority=immediate 的命令风险不得是 high`, ].join("\n"); const RCA_PROMPTS_KEY = "netcatty_rca_prompts_v1"; interface RcaPromptsConfig { phase1?: string; phase2?: string; } function loadRcaPrompts(): RcaPromptsConfig { try { const raw = localStorage.getItem(RCA_PROMPTS_KEY); if (raw) return JSON.parse(raw); } catch (_) {} return {}; } function phase1System(): string { const c = loadRcaPrompts(); return c.phase1?.trim() ? c.phase1 : DEFAULT_PHASE1_SYSTEM; } function phase2System(): string { const c = loadRcaPrompts(); return c.phase2?.trim() ? c.phase2 : DEFAULT_PHASE2_SYSTEM; } // ---------- 设备类型自动探测 ---------- /** * 未知 catalogId 时,发 3 条不同命令族的探测命令,根据哪条成功自动识别设备类型。 * 探测顺序: * 1. display memory → 华为/H3C 交换机/防火墙 * 2. show memory → Cisco IOS 交换机/防火墙 * 3. free -k → Linux bash * 全部失败 → catalogId 继续 undefined,AI 会用 unknown 兜底 */ async function autoDetectCatalog(hostConfig: HostConfig): Promise { const probes: Array<{ cmd: string; catalogId: string; negative?: RegExp }> = [ { cmd: "display memory", catalogId: "huawei_h3c_switch", negative: /未找到命令|not found|unknown command|unrecognized command|invalid input/i }, { cmd: "show memory", catalogId: "cisco_switch", negative: /not found|invalid input|unknown command/i }, { cmd: "free -k", catalogId: "linux", negative: /command not found|未找到命令|not found/i }, ]; for (const probe of probes) { try { const resp = await bridge().inspection!.runSingleCommand!( { hostConfig, command: probe.cmd, timeoutMs: 8000 }, ); const result = resp as any; const stdout = (result?.stdout || "") as string; const stderr = (result?.stderr || "") as string; // 1. SSH 层就失败了(离线/认证)→ 跳过所有探测 if (!result?.ok && /SSH|timeout|EAI_|ENOTFOUND|ECONNREFUSED|ECONNRESET/i.test(stderr || result?.error || "")) { console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ SSH 连接问题,跳过:", stderr || result?.error); break; } // 2. 命令不存在(中英文都匹配)→ 继续下一个 probe const allOutput = stdout + "\n" + stderr; if (probe.negative && probe.negative.test(allOutput)) { console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ 命令不存在:", allOutput.slice(0, 80).replace(/[\r\n]+/g, " | ")); continue; } // 3. ok=true 且输出有实质内容(>30 字符,不是只有 banner)→ 确认 catalog // Linux free -k 输出以 "Mem:" "Swap:" 开头 // 交换机 display memory 输出包含 "Total Memory" 或 "Memory Size" // 如果输出里有明显的 "未找到命令" "command not found" 就 continue 下一个 const okAndHasContent = result?.ok === true && stdout.length > 30 && !probe.negative.test(allOutput); if (okAndHasContent) { console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ success, catalogId =", probe.catalogId, "stdout sample:", stdout.slice(0, 150).replace(/[\r\n]+/g, " | ")); return probe.catalogId; } } catch (e) { console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ error:", (e as Error)?.message); break; } } console.log("[RCA] autoDetectCatalog: all probes failed, catalogId stays undefined"); return undefined; } // ---------- Fallback 命令(无 AI / AI 失败时用) ---------- function pickFallbackCmds(alert: AlertRecord, catalogId?: string): string[] { const offline = alert.rule_id === "rule_offline"; const cpu = alert.metric === "cpu_usage"; const mem = alert.metric === "memory_usage"; const disk = alert.metric === "disk_usage"; const isSwitch = /switch|huawei|h3c|cisco/i.test(catalogId || ""); if (offline) return ["ping -c 3 127.0.0.1", "ssh -V 2>&1"]; if (isSwitch) { if (cpu) return ["display cpu-usage"]; if (mem) return ["display memory"]; if (disk) return ["display device"]; return ["display version"]; } if (cpu) return ["top -bn1 | head -20", "vmstat 1 2"]; if (mem) return ["free -m", "cat /proc/meminfo | head -20"]; if (disk) return ["df -h", "iostat -x 1 1 2>/dev/null || echo no_iostat"]; return ["uname -a && uptime && ps aux --sort=-%cpu | head -5"]; } // ---------- 构建 hostConfig(renderer 侧解密 vault hosts + keys)---------- const DEVICE_EXTENSION_KEY = "netcatty_inspection_device_extension_v1"; const INSPECTION_RESULTS_KEY = "netcatty_inspection_inspection_results_v1"; /** * 从 localStorage 读 inspection device extension 拿 catalog_id(renderer 直接读,不走主进程 handler)。 * 这是 catalogId 的终极兜底 —— 因为 inspectionAgentHandlers 用的 storage adapter 和巡检调度器不同, * 会读不到 extension,这里 renderer 直接从 localStorage 拿最可靠。 */ function localCatalogIdOf(deviceId: string): string | undefined { try { // 1. 最新巡检结果里的 catalog_id(改之前存的老 result 没有,改之后新巡检会存) const resultsRaw = localStorage.getItem(INSPECTION_RESULTS_KEY); console.log("[RCA] localStorage.getItem", INSPECTION_RESULTS_KEY, "→", resultsRaw ? resultsRaw.slice(0, 200) + "..." : "null"); if (resultsRaw) { const all = JSON.parse(resultsRaw); if (Array.isArray(all)) { // 从新到旧找这个 deviceId 的第一条有 catalog_id 的 for (let i = all.length - 1; i >= 0; i--) { const r = all[i]; if (r?.device_id === deviceId && r?.catalog_id) { console.log("[RCA] found catalogId in inspection_results:", r.catalog_id, "device_id:", r.device_id); return r.catalog_id; } } } } // 2. device extension(这个始终有,只要用户在巡检中心给设备选过类型) const extRaw = localStorage.getItem(DEVICE_EXTENSION_KEY); console.log("[RCA] localStorage.getItem", DEVICE_EXTENSION_KEY, "→", extRaw ? extRaw.slice(0, 300) + "..." : "null"); if (extRaw) { const all = JSON.parse(extRaw); console.log("[RCA] device_extension keys:", Object.keys(all || {})); const ext = all?.[deviceId]; console.log("[RCA] device_extension[", deviceId, "] →", ext); if (ext?.catalog_id) { console.log("[RCA] found catalogId in device_extension:", ext.catalog_id); return ext.catalog_id; } } // 3. 遍历 device_extension 所有 key 找 host_id 匹配(用户可能是用 host_id 存的不是 device_id) if (extRaw) { const all = JSON.parse(extRaw); for (const [key, val] of Object.entries(all || {})) { const v: any = val; if (v?.host_id === deviceId || key === deviceId) { console.log("[RCA] matched ext by key/host_id:", key, v?.catalog_id); if (v?.catalog_id) return v.catalog_id; } } } } catch (e) { console.warn("[RCA] localCatalogIdOf error:", e?.message); } console.log("[RCA] localCatalogIdOf returned undefined for", deviceId); return undefined; } interface HostConfig { hostname: string; port: number; username: string; password?: string; privateKeyContent?: string; passphrase?: string; label?: string; } /** * 从 localStorage 读取 vault hosts + keys,解密后构建完整 SSH 连接配置。 * 这是 AI RCA 独立建 SSH 的关键 — 不走已有终端 session,直接从 vault 拿凭据。 */ async function buildHostConfig(deviceId: string): Promise { try { const hostsRaw = localStorage.getItem(STORAGE_KEY_HOSTS); const keysRaw = localStorage.getItem(STORAGE_KEY_KEYS); if (!hostsRaw) return null; const hostsEnc = JSON.parse(hostsRaw); const keysEnc = keysRaw ? JSON.parse(keysRaw) : []; const hosts = await decryptHosts(hostsEnc); const keys = await decryptKeys(keysEnc); const host = hosts.find((h: any) => h.id === deviceId); if (!host) return null; const cfg: HostConfig = { hostname: host.hostname, port: host.port || 22, username: host.username, label: host.label || host.hostname, }; if (host.password) cfg.password = host.password; // 私钥 const identityId = host.identityFileId; if (identityId) { const key = keys.find((k: any) => k.id === identityId); if (key?.privateKey) { cfg.privateKeyContent = key.privateKey; if (key.passphrase) cfg.passphrase = key.passphrase; } } if (!cfg.hostname || !cfg.username) return null; return cfg; } catch (_) { return null; } } // ---------- Main entry ---------- export async function runRCA(alert: AlertRecord, opts: RunRCAOptions = {}): Promise { const t0 = Date.now(); const invocations: RunRCAResult["toolInvocations"] = []; // Pre-fetch metrics (both AI path and fallback need them) let metrics: any = null; let catalogId: string | undefined; try { const r = await rpc("netcatty/inspectionHostMetrics", { deviceId: alert.device_id }); metrics = r; catalogId = (r as any)?.catalogId as string | undefined; invocations.push({ tool: "inspectionHostMetrics", ok: true, durationMs: 0 }); } catch (_) {} // Metrics 没有 catalogId(设备离线无巡检数据)→ 从 Vault + device extension 兜底 if (!catalogId) { try { const info = await rpc("netcatty/inspectionHostInfo", { deviceId: alert.device_id }); catalogId = (info as any)?.catalogId as string | undefined; if (metrics && !metrics.catalogId && catalogId) metrics = { ...metrics, catalogId }; invocations.push({ tool: "inspectionHostInfo", ok: true, durationMs: 0 }); } catch (_) {} } // 终极兜底:renderer 直接从 localStorage 读 inspection device extension if (!catalogId) { catalogId = localCatalogIdOf(alert.device_id); if (metrics && !metrics.catalogId && catalogId) metrics = { ...metrics, catalogId }; } console.log("[RCA] alert.device_id:", alert.device_id, "| catalogId:", catalogId, "| metrics?.catalogId:", metrics?.catalogId); // 终极兜底:如果所有上游都没拿到 catalogId,发 3 条探测命令自动识别设备类型 if (!catalogId) { try { const hostConfig = await buildHostConfig(alert.device_id); if (hostConfig) { catalogId = await autoDetectCatalog(hostConfig); console.log("[RCA] autoDetectCatalog result:", catalogId); if (catalogId) { if (metrics && !metrics.catalogId) metrics = { ...metrics, catalogId }; } } } catch (_) {} } // Try AI path if (!opts.forceDeterministic) { const aiResult = await runAIPath(alert, metrics, invocations, catalogId); if (aiResult) { aiResult.durationMs = Date.now() - t0; return aiResult; } } // Deterministic fallback const rca = await runFallbackPath(alert, metrics, invocations); return { success: true, source: "fallback", rca, durationMs: Date.now() - t0, toolInvocations: invocations, }; } // ---------- AI two-phase ---------- async function runAIPath( alert: AlertRecord, metrics: any, invocations: RunRCAResult["toolInvocations"], catalogIdOverride?: string, ): Promise { const provider = await resolveProvider(); if (!provider) { console.info("[RCA] No AI provider configured, using deterministic fallback"); return null; } const evidence: RCAResult["evidence"] = []; const online = metrics?.metrics?.online; const catalogId = catalogIdOverride || metrics?.catalogId as string | undefined; const catalogLabel = catalogLabelOf(catalogId); // --- Phase 1: AI 规划诊断命令 --- const phase1User = [ `## 设备型号(关键!请严格按此选命令)`, `- catalogId: ${catalogId || "unknown"}`, `- 设备类型: ${catalogLabel}`, `- 是否在线: ${online}`, ``, `## 告警信息`, `- 设备: ${alert.device_name || alert.device_id} (id=${alert.device_id})`, `- 规则: ${alert.rule_name} (${alert.rule_id})`, `- 级别: ${alert.level}`, `- 指标: ${alert.metric} = ${alert.metric_value} (阈值 ${alert.threshold})`, `- 触发时间: ${new Date(alert.triggered_at).toLocaleString("zh-CN")}`, `- 消息: ${alert.message}`, ``, `## 最新巡检 metrics`, metrics?.metrics ? JSON.stringify(metrics.metrics, null, 2) : "(暂无数据)", ``, `请输出需要执行的诊断命令列表(JSON)。`, ].join("\n"); const phase1Raw = await aiChatCompletion(provider, phase1System(), [{ role: "user", content: phase1User }]); if (!phase1Raw) return null; let plan: { commands: Array<{ command: string; purpose: string }> } | null = null; try { const parsed = parseFirstJSON(phase1Raw); if (parsed && Array.isArray(parsed.commands)) plan = parsed; } catch (_) {} if (!plan) return null; // AI 没输出有效计划 // --- 构建 hostConfig(从 vault 解密凭据,独立 SSH)--- const hostConfig = await buildHostConfig(alert.device_id); console.log("[RCA] buildHostConfig result:", hostConfig ? `hostname=${hostConfig.hostname} user=${hostConfig.username}` : "null"); if (!hostConfig) { evidence.push({ tool: "inspectionRunSingleCommand", summary: "无法执行命令(vault 中找不到设备凭据)", keyFindings: ["提示:请先在 Vault 中配置该设备的主机和凭据"], }); } // --- Execute plan via independent SSH(独立建 SSH,不依赖已有终端)--- const executedCmds: Array<{ command: string; output: string; ok: boolean; purpose?: string }> = []; for (const c of plan.commands) { if (!hostConfig) { executedCmds.push({ command: c.command, output: "skipped — 无 hostConfig", ok: false, purpose: c.purpose, }); continue; } const start = performance.now(); try { console.log("[RCA] exec cmd:", c.command, "on", hostConfig.hostname); // 优先走 preload inspection 命名空间,兜底直接走 ipcRenderer.invoke const b = bridge(); let resp: any = null; if (b.inspection?.runSingleCommand) { resp = await b.inspection.runSingleCommand({ hostConfig, command: c.command, timeoutMs: 15000, }); } else { // 兜底:直接调 ipcRenderer(preload 没注册时可能走这里) const w = window as any; if (w.netcatty?.ipcRenderer?.invoke) { resp = await w.netcatty.ipcRenderer.invoke("inspection:runSingleCommand", { hostConfig, command: c.command, timeoutMs: 15000, }); } else { throw new Error("inspection.runSingleCommand 不可用"); } } const rawOut = (resp?.stdout || "") + (resp?.stderr ? "\n--- stderr ---\n" + resp.stderr : ""); const outStr = _stripVendorBanner(rawOut).slice(0, 5000); const rawFirst200 = rawOut.slice(0, 200).replace(/\n/g, "\\n"); const cleanFirst200 = outStr.slice(0, 200).replace(/\n/g, "\\n"); console.log("[RCA] cmd result:", resp); if (rawFirst200 !== cleanFirst200) { console.log("[RCA] vendor banner stripped:", { raw: rawFirst200, clean: cleanFirst200 }); } executedCmds.push({ command: c.command, output: outStr, ok: true, purpose: c.purpose, }); invocations.push({ tool: "inspectionRunCommand", ok: true, durationMs: Math.round(performance.now() - start), }); } catch (e: any) { executedCmds.push({ command: c.command, output: e?.message || String(e), ok: false, purpose: c.purpose, }); invocations.push({ tool: "inspectionRunCommand", ok: false, durationMs: Math.round(performance.now() - start), }); } } // --- Phase 2: AI 综合 RCA --- const phase2User = [ `## 设备信息`, `- catalogId: ${catalogId || "unknown"} (${catalogLabel})`, `- 是否在线: ${online}`, ``, `## 告警信息`, `- 设备: ${alert.device_name || alert.device_id}`, `- 规则: ${alert.rule_name} (${alert.rule_id})`, `- 指标: ${alert.metric} = ${alert.metric_value} (阈值 ${alert.threshold})`, `- 级别: ${alert.level}`, ``, `## 最新巡检 metrics`, metrics?.metrics ? JSON.stringify(metrics.metrics, null, 2) : "(暂无数据)", ``, `## 诊断命令执行结果`, ...executedCmds.map( (c) => `### ${c.command}${c.ok ? "" : " ❌ failed"}\n` + `用途: ${c.purpose || "-"}\n` + `\`\`\`\n${c.output.slice(0, 2500)}\n\`\`\``, ), ``, `请综合以上证据分析根因并输出 JSON。`, ].join("\n"); const phase2Raw = await aiChatCompletion(provider, phase2System(), [{ role: "user", content: phase2User }]); if (!phase2Raw) return null; const rca = parseFinalRCA(phase2Raw, alert, evidence); if (!rca) return null; // Attach command outputs to evidence for (const c of executedCmds) { evidence.push({ tool: c.command, summary: c.purpose || "诊断命令", keyFindings: [c.output.slice(0, 500)], }); } rca.evidence = evidence; return { success: true, source: "ai", rca, rawText: phase2Raw, durationMs: 0, toolInvocations: invocations, }; } function parseFirstJSON(text: string): any { let s = text.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/i, ""); try { return JSON.parse(s); } catch (_) {} const m = s.match(/\{[\s\S]*\}/); if (m) { try { return JSON.parse(m[0]); } catch (_) {} } return null; } function parseFinalRCA(text: string, alert: AlertRecord, evidence: RCAResult["evidence"]): RCAResult | null { const parsed = parseFirstJSON(text); if (!parsed || typeof parsed !== "object") return null; if (!parsed.rootCauses?.length && !parsed.summary) return null; return { alertId: alert.id, deviceId: alert.device_id, deviceName: alert.device_name || alert.device_id, confidence: Math.max(0, Math.min(100, Number(parsed.confidence) || 0)), rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses.map(String) : [], evidence, recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations.map((r: any) => ({ priority: ["immediate", "short_term", "long_term"].includes(r?.priority) ? r.priority : "short_term", action: String(r?.action ?? ""), command: r?.command ? String(r.command) : undefined, risk: ["low", "medium", "high"].includes(r?.risk) ? r.risk : "medium", note: r?.note ? String(r.note) : undefined, })) : [], summary: String(parsed.summary ?? ""), }; } // ---------- Deterministic ---------- async function runFallbackPath( alert: AlertRecord, metrics: any, invocations: RunRCAResult["toolInvocations"], ): Promise { const evidence: RCAResult["evidence"] = []; const commands = pickFallbackCmds(alert, metrics?.catalogId as string | undefined); for (const cmd of commands) { const start = performance.now(); try { const resp = (await rpc("netcatty/inspectionRunCommand", { deviceId: alert.device_id, command: cmd, timeoutMs: 15000, })) as any; const outStr = (resp?.stdout || "") + (resp?.stderr ? "\n--- stderr ---\n" + resp.stderr : ""); evidence.push({ tool: cmd, summary: "诊断命令输出", keyFindings: [outStr.slice(0, 2000)] }); invocations.push({ tool: "inspectionRunCommand", ok: true, durationMs: Math.round(performance.now() - start) }); } catch (e: any) { evidence.push({ tool: cmd, summary: "命令执行失败", keyFindings: [e?.message || ""] }); invocations.push({ tool: "inspectionRunCommand", ok: false, durationMs: Math.round(performance.now() - start) }); } } const offline = alert.rule_id === "rule_offline"; const cpu = alert.metric === "cpu_usage"; const mem = alert.metric === "memory_usage"; const disk = alert.metric === "disk_usage"; const hostLabel = alert.device_name || alert.device_id; const rootCauses: string[] = []; const recs: RCAResult["recommendations"] = []; let confidence = 40; if (offline) { rootCauses.push("设备离线或 SSH 连接异常"); confidence = 55; recs.push({ priority: "immediate", action: "检查网络连通性", command: `ping -c 3 ${hostLabel}`, risk: "low" }); recs.push({ priority: "immediate", action: "确认 SSH 服务和端口", risk: "low" }); } if (cpu) { rootCauses.push(`CPU ${alert.metric_value}% 超过阈值 ${alert.threshold}% — 热点进程或 I/O wait`); confidence = 60; recs.push({ priority: "immediate", action: "定位 CPU 热点进程", command: "top -bn1 | head -20 && ps aux --sort=-%cpu | head -10", risk: "low" }); } if (mem) { rootCauses.push(`内存 ${alert.metric_value}% — 内存泄漏或突发负载`); confidence = 60; recs.push({ priority: "immediate", action: "检查 top 内存进程", command: "free -m && ps aux --sort=-%mem | head -10", risk: "low" }); } if (disk) { rootCauses.push("磁盘使用率超限 — 日志堆积或大文件残留"); confidence = 60; recs.push({ priority: "immediate", action: "定位大文件并清理", command: "df -h && du -sh /var/log/* /tmp/* | sort -rh | head -20", risk: "medium", note: "删除前确认不影响日志轮转" }); } if (!rootCauses.length) { rootCauses.push("告警触发但信息不足,需要人工排查"); confidence = 25; recs.push({ priority: "short_term", action: "人工登录设备检查", risk: "low" }); } return { alertId: alert.id, deviceId: alert.device_id, deviceName: hostLabel, confidence, rootCauses, evidence, recommendations: recs, summary: `${hostLabel} 触发 ${alert.level} 级告警(${alert.rule_name})。根因: ${rootCauses[0]}。置信度 ${confidence}%(确定性分析,无 AI 参与)。`, }; } export function parseRCAResult(raw: string, alert: AlertRecord): RCAResult | null { return parseFinalRCA(raw, alert, []); }