Files
SenMeshNet/infrastructure/ai/rcaAgent/rcaRunner.ts
zhaolei 2f9f8da48e
Some checks failed
AI automation / Route event (push) Has been cancelled
AI automation / Hand reopened issue to maintainers (push) Has been cancelled
AI automation / Clean source issue state (push) Has been cancelled
AI automation / Reconcile handoffs (push) Has been cancelled
AI automation / Classify issue (push) Has been cancelled
AI automation / Claude Code smoke (push) Has been cancelled
AI automation / Review issue follow-up (push) Has been cancelled
AI automation / Publish issue follow-up (push) Has been cancelled
AI automation / Implement with Claude Code (push) Has been cancelled
AI automation / Publish implement PR (push) Has been cancelled
AI automation / Continue queued issue comments (push) Has been cancelled
AI automation / Codex review loop (push) Has been cancelled
AI automation / Publish Codex fix (push) Has been cancelled
AI automation / Clear Codex dispatch marker (push) Has been cancelled
AI automation / Own PR re-request Codex (push) Has been cancelled
AI automation / External PR re-request Codex (push) Has been cancelled
AI automation / Poll Codex reaction / retry (push) Has been cancelled
Initial commit: NetMesh smart ops platform with 3D topology, inspection reports, RCA and AI ops assistant
2026-10-15 13:53:04 +08:00

938 lines
38 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* RCA (Root Cause Analysis) runner — v2 AI-driven.
*
* Two-phase design (no AI SDK tool-loop required):
*
* Phase 1 — Plan:
* AI receives alert + context + system prompt (diagnosis workflow).
* Outputs a JSON plan: which diagnosis commands to run on which device.
*
* Execute — We run the plan:
* For each command → aiCapability("netcatty/inspectionRunCommand", { deviceId, command })
* Collect stdout into evidence bundle.
*
* Phase 2 — Synthesize:
* AI receives alert + metrics + ALL command outputs.
* Outputs final RCAResult JSON (root causes + recommendations).
*
* Deterministic fallback — if any phase fails, we use local heuristics
* (inspection.runCommand still runs real SSH commands).
*/
import type { AlertRecord } from "../../inspection/inspectionApi";
import {
STORAGE_KEY_HOSTS,
STORAGE_KEY_KEYS,
} from "../../config/storageKeys";
import {
decryptHosts,
decryptKeys,
} from "../../persistence/secureFieldAdapter";
export interface RunRCAOptions {
forceDeterministic?: boolean;
onTool?: (name: string, ok: boolean, summary: string) => void;
}
export interface RCAResult {
alertId: string;
deviceId: string;
deviceName: string;
confidence: number;
rootCauses: string[];
evidence: Array<{
tool: string;
summary: string;
keyFindings: string[];
}>;
recommendations: Array<{
priority: "immediate" | "short_term" | "long_term";
action: string;
command?: string;
risk: "low" | "medium" | "high";
note?: string;
}>;
summary: string;
}
export interface RunRCAResult {
success: boolean;
rca?: RCAResult;
rawText?: string;
source: "ai" | "fallback";
error?: string;
durationMs: number;
toolInvocations: Array<{ tool: string; ok: boolean; durationMs: number }>;
}
// ---------- Bridge (renderer preload) ----------
type Bridge = {
aiCapability?: (
rpcMethod: string,
params: Record<string, unknown>,
chatSessionId?: string,
) => Promise<unknown>;
aiFetch?: (
url: string, method: string, headers: Record<string, string>,
body?: string, providerId?: string, skipHostCheck?: boolean,
followRedirects?: boolean, skipTLSVerify?: boolean, timeoutMs?: number,
) => Promise<{ ok: boolean; status: number; data: string; error?: string }>;
aiAllowlistAddHost?: (baseURL: string) => Promise<unknown>;
aiSyncProviders?: (providers: unknown[]) => Promise<unknown>;
inspection?: {
runSingleCommand?: (payload: {
hostConfig: HostConfig;
command: string;
timeoutMs?: number;
}) => Promise<any>;
};
};
function bridge(): Bridge {
return (window as unknown as { netcatty?: Bridge }).netcatty ?? ({} as Bridge);
}
async function rpc<T>(method: string, params: Record<string, unknown>): Promise<T> {
const b = bridge();
if (!b.aiCapability) throw new Error("aiCapability bridge unavailable");
const result = await b.aiCapability(method, params);
// aiCapability returns handler result directly — if ok: false, surface error
if (result && typeof result === "object" && (result as any).ok === false) {
throw new Error((result as any).error || "RPC failed");
}
return result as T;
}
async function safeRpc<T>(method: string, params: Record<string, unknown>): Promise<T | null> {
try { return await rpc<T>(method, params); } catch (_) { return null; }
}
// ---------- AI chat completions via aiFetch ----------
interface ProviderInfo {
providerId: string;
apiKey: string;
baseUrl: string;
model: string;
style?: string;
}
// Storage keys (keep in sync with infrastructure/config/storageKeys.ts)
const AI_PROVIDERS_KEY = "netcatty_ai_providers_v1";
const AI_ACTIVE_PROVIDER_KEY = "netcatty_ai_active_provider_v1";
async function resolveProvider(): Promise<ProviderInfo | null> {
let providers: any[] | null = null;
let activeId: string | null = null;
try {
const raw = localStorage.getItem(AI_PROVIDERS_KEY);
if (raw) providers = JSON.parse(raw);
activeId = localStorage.getItem(AI_ACTIVE_PROVIDER_KEY);
} catch (_) {}
if (!Array.isArray(providers) || providers.length === 0) return null;
// Prefer active provider, fall back to first enabled one
let primary: any = null;
if (activeId) primary = providers.find((p) => p.id === activeId);
if (!primary) primary = providers.find((p) => p.enabled !== false);
if (!primary?.apiKey) return null;
// 同步 providers 到主进程(主进程解密 + host allowlist 都依赖它)
// Catty SDK 正常流程会自动 sync,RCA 跳过了所以这里手动补一次
const b = bridge();
if (b.aiSyncProviders) {
try { await b.aiSyncProviders(providers); } catch (_) {}
}
const baseUrl = primary.baseURL || "";
const style = primary.style || detectStyle(baseUrl || primary.providerId || "");
const path = style === "anthropic" ? "/v1/messages"
: style === "google" ? "/v1beta/models"
: "/chat/completions";
return {
providerId: primary.id || "default",
apiKey: primary.apiKey, // 可能是 enc:v1: 加密 placeholder,主进程 injectApiKeyIntoRequest 会解密替换
baseUrl: (baseUrl.replace(/\/$/, "")) + path,
model: primary.defaultModel || "gpt-4o-mini",
style,
};
}
/** 清洗厂商登录 banner / URL / 版权噪音,让 AI 只看到命令输出 */
function _stripVendorBanner(text: string): string {
if (!text) return text;
let out = text;
// 1. 删所有 http/https URL
out = out.replace(/https?:\/\/[^\s|,,。]+/g, "");
// 2. 删裸域名 www.xxx.com / 管理门户路径
out = out.replace(/\bwww\.[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?/gi, "");
out = out.replace(/\bmanagement\.[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?/gi, "");
// 3. 删残留的 Web 脚本后缀片段(banner 里可能出现裸 .aspx / .do 等)
out = out.replace(/\.(aspx|do|cgi|php|jsp|html?|shtml|asp|cfm|jspx|action)(\/[^\s]*)?\b/gi, "");
// 4. 删整行噪音
out = out
.split("\n")
.filter(line => {
const t = line.trim();
if (!t) return true;
// SSH 登录 banner
if (/Welcome to (Kylin|Ubuntu|CentOS|Red Hat|RedHat|Debian|SUSE|openSUSE|Fedora|AlmaLinux|Rocky Linux)/i.test(t)) return false;
if (/^(Huawei|H3C|Cisco|Ruijie|ZTE|锐捷|中兴|华为)/.test(t) && /(Platform|Software|Operating System|Comware|VRP)/i.test(t)) return false;
// 管理门户 / 版权 / 警告
if (/^\s*\*?\s*Management\s*[::]/i.test(t)) return false;
if (/^\s*\*?\s*(Copyright|Trademark|All Rights Reserved|For Internal Use|Unauthorized)/i.test(t)) return false;
if (/^\s*\*{5,}\s*$/.test(t)) return false;
// SSH MOTD(登录欢迎信息)
if (/^Last login:\s/i.test(t)) return false;
if (/^You have (new )?mail/i.test(t)) return false;
if (/^No mail$/i.test(t)) return false;
if (/^MOTD(\s|$)/i.test(t)) return false;
if (/^Message of the Day/i.test(t)) return false;
// Kylin / Ubuntu 的欢迎行(如 "Welcome to Kylin V10 SP1..." 已在上面,这里兜底)
if (/^\s*\*?\s*(Welcome|欢迎)/i.test(t) && /(Kylin|Ubuntu|CentOS|Linux|操作系统)/i.test(t)) return false;
// 麒麟 OS 的软件协议 / 许可提示
if (/^\s*\*?\s*(License|许可|授权|Service Pack|SP\d+)/i.test(t) && /(Kylin|麒麟)/i.test(t)) return false;
return true;
})
.join("\n");
// 5. 多空行压缩
out = out.replace(/\n{3,}/g, "\n\n").trim();
return out;
}
function detectStyle(baseUrlOrId: string): string {
const lower = baseUrlOrId.toLowerCase();
if (/anthropic|claude/.test(lower)) return "anthropic";
if (/google|gemini/.test(lower)) return "google";
return "openai";
}
const CATALOG_LABELS: Record<string, string> = {
linux: "Linux 服务器",
linux_server: "Linux 服务器",
windows: "Windows 服务器",
huawei_h3c_switch: "华为/H3C 交换机",
cisco_switch: "Cisco 交换机",
huawei_firewall: "华为防火墙",
fortinet_firewall: "飞塔防火墙",
};
function catalogLabelOf(catalogId?: string): string {
if (!catalogId) return "未知(unknown)";
return CATALOG_LABELS[catalogId] || catalogId;
}
async function aiChatCompletion(
provider: ProviderInfo,
system: string,
messages: Array<{ role: "user" | "assistant"; content: string }>,
): Promise<string> {
const b = bridge();
if (!b.aiFetch) throw new Error("aiFetch bridge unavailable");
// 预注册 host 到主进程 allowlist(独立 RCA 路径不走 Catty SDK,没同步过 provider host)
// 幂等调用 — host 已存在时只是刷新 TTL
if ((b as any).aiAllowlistAddHost) {
try { await (b as any).aiAllowlistAddHost(provider.baseUrl); } catch (_) {}
}
const body = JSON.stringify({
model: provider.model,
messages: [
{ role: "system", content: system },
...messages,
],
temperature: 0,
stream: false,
});
const headers: Record<string, string> = {
"Content-Type": "application/json",
// 用 placeholder — 主进程 injectApiKeyIntoRequest 会用 providerId 解密注入真实 key
Authorization: `Bearer __IPC_SECURED__`,
};
if (provider.style === "anthropic") {
headers["anthropic-version"] = "2023-06-01";
}
// RCA 分析也是多轮思考 + 大输出,给 120s timeout
const resp = await b.aiFetch(
provider.baseUrl, "POST", headers, body, provider.providerId,
/*skipHostCheck*/ true, /*followRedirects*/ false, /*skipTLSVerify*/ false,
/*timeoutMs*/ 1200000,
);
if (!resp.ok) throw new Error(`AI ${resp.status}: ${resp.data?.slice(0, 200) || resp.error}`);
return parseAiChat(resp.data || "{}", provider.style);
}
function parseAiChat(data: string, style?: string): string {
const parsed = JSON.parse(data || "{}");
if (style === "anthropic") return parsed.content?.[0]?.text ?? "";
if (style === "google") return parsed.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
return parsed.choices?.[0]?.message?.content ?? "";
}
// ---------- System prompts ----------
// 内置默认值 —— 存储在 STORAGE_KEY_RCA_PROMPTS 里的自定义 prompt 优先使用
export const DEFAULT_PHASE1_SYSTEM = [
`你是 NetMesh 运维平台的 AI 诊断规划专家。`,
`平台会在设备上执行你规划的每条命令,收集 stdout 返回给你。`,
`如果设备离线(online=false),不要跑任何需要 SSH 的命令 — 改成先确认网络可达性。`,
``,
`## 设备 Catalog 与命令族(必须严格匹配)`,
`| catalogId | 设备类型 | Shell | 实际巡检命令(系统已在用) | 推荐诊断命令 |`,
`|---|---|---|---|---|`,
`| linux | Linux 服务器 | bash/sh | top -bn1 / mpstat 1 1 / free -k / cat /proc/meminfo / df -h | top -bn1 \| head -20, vmstat 1 2, free -m, cat /proc/meminfo \| head -20, df -h, ps aux --sort=-%cpu \| head -10, iostat -x 1 1 2>/dev/null \|\| echo no_iostat, lsof -i, journalctl -xe --no-pager \| tail -50, dmesg \| tail -50 |`,
`| windows | Windows 服务器 | PowerShell | Get-WmiObject Win32_Processor / Win32_OperatingSystem / Win32_LogicalDisk / Get-Service | Get-Process \| Sort-Object CPU -Descending \| Select-Object -First 10, Get-CimInstance Win32_OperatingSystem, Get-CimInstance Win32_LogicalDisk, Get-Service \| Where-Object {$_.Status -ne 'Running'}, Get-EventLog -LogName System -Newest 50 |`,
`| huawei_h3c_switch | 华为/H3C 交换机 | HUAWEI CLI | display cpu-usage / display memory / display interface brief / display power / display fan | **禁止 display cpu-usage(有刷新动画抓不到完整输出)**;可用命令: display version, display device, display memory, display power, display fan, display logbuffer, display stp brief, display vrrp, display vlan brief, display arp |`,
`| cisco_switch | Cisco 交换机 | IOS CLI | 复用华为交换机巡检(暂无专用 adapter) | show version, show processes cpu sorted, show memory summary, show interfaces brief, show interfaces counters errors, show inventory, show log tail 50, show vlan brief, show spanning-tree summary |`,
`| huawei_firewall | 华为防火墙 | HUAWEI CLI | 复用华为交换机巡检(暂无专用 adapter) | display version, display cpu-usage, display memory, display firewall session table, display current-configuration, display security-policy count, display server-protect statistics |`,
`| fortinet_firewall | 飞塔防火墙 | FortiOS CLI | 复用华为交换机巡检(暂无专用 adapter) | get system status, get system performance status, get system performance cpu, get system performance memory, get firewall session list, get config route static, diagnose debug flow stat |`,
``,
`## 输出格式(严格 JSON,不要 markdown 代码块之外的文字)`,
`{`,
` "commands": [`,
` { "command": "display cpu-usage", "purpose": "查看设备 CPU 使用率" }`,
` ],`,
` "reasoning": "一句话说明为什么选这些命令"`,
`}`,
``,
`## 命令规划规则`,
`- **命令必须严格从上面"推荐诊断命令"列中选择**,不得臆造未知命令(如 display process、display memory-usage)`,
`- **严禁输出任何外部 URL / 厂商官网链接 / 技术文档链接**(如 management.h3c.com、www.kylinos.cn、cisco.com/wiki、forum.huawei 等) —— 你的运行环境在内网,外网链接无法访问,输出它们没有任何意义`,
`- **严禁引用厂商文档名称和章节号**(如"参见 H3C S5700 V200R010 配置指南"),设备 firmware / 型号信息必须从诊断命令输出中提取`,
`- **严禁推荐访问外网工具**(如 curl/wget 拉远程脚本、apt/yum 从公网仓库安装)`,
`- **禁止选择语义重复的命令**(如 display memory 和 display memory-usage 只需选一条)`,
`- **禁止选择持续刷新/交互式命令** — display cpu-usage 有动态刷新动画,无法完整抓取输出。请改用 display device(含 CPU 模块状态)或 display power`,
`- **关闭分页的前置命令**(让 display logbuffer 完整输出):华为/H3C 用 "screen-length 0 temporary"(用户视图下可用),不要用 "terminal length 0"(那是 system-view 下的)。独立作为 commands 数组里的第一条`,
`- **display logbuffer** 建议用 "display logbuffer level warning" 或 "display logbuffer size 512" 缩小输出范围,避免 10000+ 行日志`,
`- display version 是只读静态信息,建议放在 plan 里至少一次(用来识别设备型号/固件版本,对根因分析很关键)`,
`- CPU 告警 → 至少 2 条命令(display device 看硬件状态 + display logbuffer 查告警)`,
`- 内存告警 → 至少 2 条命令(display memory + display logbuffer level warning 查 OOM)`,
`- 离线告警 → 只跑连通性命令(ping, traceroute),不要尝试 SSH 命令`,
`- 不超过 6 条命令,总耗时不超过 60 秒`,
`- 命令必须简短可执行,不含交互式程序、不含 tail -f 等持续输出`,
`- 如果 catalogId 是 unknown,从设备名称 / metrics / 告警特征推断,拿不准时先跑 uname -a(Linux)或 display version(华为)或 show version(Cisco)`,
].join("\n");
export const DEFAULT_PHASE2_SYSTEM = [
`你是 NetMesh 智慧运维平台的 AI 根因分析专家。`,
`你收到了一条告警 + 最新巡检 metrics + 多条诊断命令的执行输出。`,
`请综合这些证据,给出结构化的根因分析和处置建议。`,
``,
`## 核心约束 — 内网环境,严格执行`,
`- 严禁输出任何外部 URL / 厂商官网链接 / 技术文档链接(如 management.h3c.com、www.kylinos.cn、cisco.com/wiki 等)`,
`- 严禁引用厂商文档名称、章节号、版本号(如"H3C S5700 V200R010 配置指南 第 3 章")`,
`- 严禁推荐访问外网的工具或步骤(如"从官网下载固件"、"curl 远程脚本"、"apt 公网仓库安装")`,
`- 设备固件版本、硬件型号等信息必须从诊断命令的 stdout 输出中提取,不得脑补厂商数据库内容`,
`- 所有处置建议必须在**不依赖外网访问**的前提下可执行`,
``,
`## 输出格式(严格 JSON,不要 markdown 代码块)`,
`{`,
` "confidence": 85,`,
` "rootCauses": ["最可能根因", "次要可能性"],`,
` "summary": "2-3 句摘要,适合运维人员快速阅读",`,
` "recommendations": [`,
` { "priority": "immediate|short_term|long_term", "action": "要做什么", "command": "可选命令", "risk": "low|medium|high", "note": "注意事项" }`,
` ]`,
`}`,
``,
`## 规则`,
`- confidence 基于证据充分性(有 metrics + 命令输出且相互印证 → 80+;只有 metrics → 60;无数据 → 30 以下)`,
`- recommendations 至少 2 条,最多 5 条,按紧急程度排序`,
`- command 只放确定安全的命令,破坏性操作只在 action 里描述`,
`- priority=immediate 的命令风险不得是 high`,
].join("\n");
const RCA_PROMPTS_KEY = "netcatty_rca_prompts_v1";
interface RcaPromptsConfig {
phase1?: string;
phase2?: string;
}
function loadRcaPrompts(): RcaPromptsConfig {
try {
const raw = localStorage.getItem(RCA_PROMPTS_KEY);
if (raw) return JSON.parse(raw);
} catch (_) {}
return {};
}
function phase1System(): string {
const c = loadRcaPrompts();
return c.phase1?.trim() ? c.phase1 : DEFAULT_PHASE1_SYSTEM;
}
function phase2System(): string {
const c = loadRcaPrompts();
return c.phase2?.trim() ? c.phase2 : DEFAULT_PHASE2_SYSTEM;
}
// ---------- 设备类型自动探测 ----------
/**
* 未知 catalogId 时,发 3 条不同命令族的探测命令,根据哪条成功自动识别设备类型。
* 探测顺序:
* 1. display memory → 华为/H3C 交换机/防火墙
* 2. show memory → Cisco IOS 交换机/防火墙
* 3. free -k → Linux bash
* 全部失败 → catalogId 继续 undefined,AI 会用 unknown 兜底
*/
async function autoDetectCatalog(hostConfig: HostConfig): Promise<string | undefined> {
const probes: Array<{ cmd: string; catalogId: string; negative?: RegExp }> = [
{ cmd: "display memory", catalogId: "huawei_h3c_switch", negative: /未找到命令|not found|unknown command|unrecognized command|invalid input/i },
{ cmd: "show memory", catalogId: "cisco_switch", negative: /not found|invalid input|unknown command/i },
{ cmd: "free -k", catalogId: "linux", negative: /command not found|未找到命令|not found/i },
];
for (const probe of probes) {
try {
const resp = await bridge().inspection!.runSingleCommand!(
{ hostConfig, command: probe.cmd, timeoutMs: 8000 },
);
const result = resp as any;
const stdout = (result?.stdout || "") as string;
const stderr = (result?.stderr || "") as string;
// 1. SSH 层就失败了(离线/认证)→ 跳过所有探测
if (!result?.ok && /SSH|timeout|EAI_|ENOTFOUND|ECONNREFUSED|ECONNRESET/i.test(stderr || result?.error || "")) {
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ SSH 连接问题,跳过:", stderr || result?.error);
break;
}
// 2. 命令不存在(中英文都匹配)→ 继续下一个 probe
const allOutput = stdout + "\n" + stderr;
if (probe.negative && probe.negative.test(allOutput)) {
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ 命令不存在:",
allOutput.slice(0, 80).replace(/[\r\n]+/g, " | "));
continue;
}
// 3. ok=true 且输出有实质内容(>30 字符,不是只有 banner)→ 确认 catalog
// Linux free -k 输出以 "Mem:" "Swap:" 开头
// 交换机 display memory 输出包含 "Total Memory" 或 "Memory Size"
// 如果输出里有明显的 "未找到命令" "command not found" 就 continue 下一个
const okAndHasContent = result?.ok === true
&& stdout.length > 30
&& !probe.negative.test(allOutput);
if (okAndHasContent) {
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ success, catalogId =", probe.catalogId,
"stdout sample:", stdout.slice(0, 150).replace(/[\r\n]+/g, " | "));
return probe.catalogId;
}
} catch (e) {
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ error:", (e as Error)?.message);
break;
}
}
console.log("[RCA] autoDetectCatalog: all probes failed, catalogId stays undefined");
return undefined;
}
// ---------- Fallback 命令(无 AI / AI 失败时用) ----------
function pickFallbackCmds(alert: AlertRecord, catalogId?: string): string[] {
const offline = alert.rule_id === "rule_offline";
const cpu = alert.metric === "cpu_usage";
const mem = alert.metric === "memory_usage";
const disk = alert.metric === "disk_usage";
const isSwitch = /switch|huawei|h3c|cisco/i.test(catalogId || "");
if (offline) return ["ping -c 3 127.0.0.1", "ssh -V 2>&1"];
if (isSwitch) {
if (cpu) return ["display cpu-usage"];
if (mem) return ["display memory"];
if (disk) return ["display device"];
return ["display version"];
}
if (cpu) return ["top -bn1 | head -20", "vmstat 1 2"];
if (mem) return ["free -m", "cat /proc/meminfo | head -20"];
if (disk) return ["df -h", "iostat -x 1 1 2>/dev/null || echo no_iostat"];
return ["uname -a && uptime && ps aux --sort=-%cpu | head -5"];
}
// ---------- 构建 hostConfig(renderer 侧解密 vault hosts + keys)----------
const DEVICE_EXTENSION_KEY = "netcatty_inspection_device_extension_v1";
const INSPECTION_RESULTS_KEY = "netcatty_inspection_inspection_results_v1";
/**
* 从 localStorage 读 inspection device extension 拿 catalog_id(renderer 直接读,不走主进程 handler)。
* 这是 catalogId 的终极兜底 —— 因为 inspectionAgentHandlers 用的 storage adapter 和巡检调度器不同,
* 会读不到 extension,这里 renderer 直接从 localStorage 拿最可靠。
*/
function localCatalogIdOf(deviceId: string): string | undefined {
try {
// 1. 最新巡检结果里的 catalog_id(改之前存的老 result 没有,改之后新巡检会存)
const resultsRaw = localStorage.getItem(INSPECTION_RESULTS_KEY);
console.log("[RCA] localStorage.getItem", INSPECTION_RESULTS_KEY, "→", resultsRaw ? resultsRaw.slice(0, 200) + "..." : "null");
if (resultsRaw) {
const all = JSON.parse(resultsRaw);
if (Array.isArray(all)) {
// 从新到旧找这个 deviceId 的第一条有 catalog_id 的
for (let i = all.length - 1; i >= 0; i--) {
const r = all[i];
if (r?.device_id === deviceId && r?.catalog_id) {
console.log("[RCA] found catalogId in inspection_results:", r.catalog_id, "device_id:", r.device_id);
return r.catalog_id;
}
}
}
}
// 2. device extension(这个始终有,只要用户在巡检中心给设备选过类型)
const extRaw = localStorage.getItem(DEVICE_EXTENSION_KEY);
console.log("[RCA] localStorage.getItem", DEVICE_EXTENSION_KEY, "→", extRaw ? extRaw.slice(0, 300) + "..." : "null");
if (extRaw) {
const all = JSON.parse(extRaw);
console.log("[RCA] device_extension keys:", Object.keys(all || {}));
const ext = all?.[deviceId];
console.log("[RCA] device_extension[", deviceId, "] →", ext);
if (ext?.catalog_id) {
console.log("[RCA] found catalogId in device_extension:", ext.catalog_id);
return ext.catalog_id;
}
}
// 3. 遍历 device_extension 所有 key 找 host_id 匹配(用户可能是用 host_id 存的不是 device_id)
if (extRaw) {
const all = JSON.parse(extRaw);
for (const [key, val] of Object.entries(all || {})) {
const v: any = val;
if (v?.host_id === deviceId || key === deviceId) {
console.log("[RCA] matched ext by key/host_id:", key, v?.catalog_id);
if (v?.catalog_id) return v.catalog_id;
}
}
}
} catch (e) {
console.warn("[RCA] localCatalogIdOf error:", e?.message);
}
console.log("[RCA] localCatalogIdOf returned undefined for", deviceId);
return undefined;
}
interface HostConfig {
hostname: string;
port: number;
username: string;
password?: string;
privateKeyContent?: string;
passphrase?: string;
label?: string;
}
/**
* 从 localStorage 读取 vault hosts + keys,解密后构建完整 SSH 连接配置。
* 这是 AI RCA 独立建 SSH 的关键 — 不走已有终端 session,直接从 vault 拿凭据。
*/
async function buildHostConfig(deviceId: string): Promise<HostConfig | null> {
try {
const hostsRaw = localStorage.getItem(STORAGE_KEY_HOSTS);
const keysRaw = localStorage.getItem(STORAGE_KEY_KEYS);
if (!hostsRaw) return null;
const hostsEnc = JSON.parse(hostsRaw);
const keysEnc = keysRaw ? JSON.parse(keysRaw) : [];
const hosts = await decryptHosts(hostsEnc);
const keys = await decryptKeys(keysEnc);
const host = hosts.find((h: any) => h.id === deviceId);
if (!host) return null;
const cfg: HostConfig = {
hostname: host.hostname,
port: host.port || 22,
username: host.username,
label: host.label || host.hostname,
};
if (host.password) cfg.password = host.password;
// 私钥
const identityId = host.identityFileId;
if (identityId) {
const key = keys.find((k: any) => k.id === identityId);
if (key?.privateKey) {
cfg.privateKeyContent = key.privateKey;
if (key.passphrase) cfg.passphrase = key.passphrase;
}
}
if (!cfg.hostname || !cfg.username) return null;
return cfg;
} catch (_) {
return null;
}
}
// ---------- Main entry ----------
export async function runRCA(alert: AlertRecord, opts: RunRCAOptions = {}): Promise<RunRCAResult> {
const t0 = Date.now();
const invocations: RunRCAResult["toolInvocations"] = [];
// Pre-fetch metrics (both AI path and fallback need them)
let metrics: any = null;
let catalogId: string | undefined;
try {
const r = await rpc<any>("netcatty/inspectionHostMetrics", { deviceId: alert.device_id });
metrics = r;
catalogId = (r as any)?.catalogId as string | undefined;
invocations.push({ tool: "inspectionHostMetrics", ok: true, durationMs: 0 });
} catch (_) {}
// Metrics 没有 catalogId(设备离线无巡检数据)→ 从 Vault + device extension 兜底
if (!catalogId) {
try {
const info = await rpc<any>("netcatty/inspectionHostInfo", { deviceId: alert.device_id });
catalogId = (info as any)?.catalogId as string | undefined;
if (metrics && !metrics.catalogId && catalogId) metrics = { ...metrics, catalogId };
invocations.push({ tool: "inspectionHostInfo", ok: true, durationMs: 0 });
} catch (_) {}
}
// 终极兜底:renderer 直接从 localStorage 读 inspection device extension
if (!catalogId) {
catalogId = localCatalogIdOf(alert.device_id);
if (metrics && !metrics.catalogId && catalogId) metrics = { ...metrics, catalogId };
}
console.log("[RCA] alert.device_id:", alert.device_id, "| catalogId:", catalogId, "| metrics?.catalogId:", metrics?.catalogId);
// 终极兜底:如果所有上游都没拿到 catalogId,发 3 条探测命令自动识别设备类型
if (!catalogId) {
try {
const hostConfig = await buildHostConfig(alert.device_id);
if (hostConfig) {
catalogId = await autoDetectCatalog(hostConfig);
console.log("[RCA] autoDetectCatalog result:", catalogId);
if (catalogId) {
if (metrics && !metrics.catalogId) metrics = { ...metrics, catalogId };
}
}
} catch (_) {}
}
// Try AI path
if (!opts.forceDeterministic) {
const aiResult = await runAIPath(alert, metrics, invocations, catalogId);
if (aiResult) {
aiResult.durationMs = Date.now() - t0;
return aiResult;
}
}
// Deterministic fallback
const rca = await runFallbackPath(alert, metrics, invocations);
return {
success: true,
source: "fallback",
rca,
durationMs: Date.now() - t0,
toolInvocations: invocations,
};
}
// ---------- AI two-phase ----------
async function runAIPath(
alert: AlertRecord,
metrics: any,
invocations: RunRCAResult["toolInvocations"],
catalogIdOverride?: string,
): Promise<RunRCAResult | null> {
const provider = await resolveProvider();
if (!provider) {
console.info("[RCA] No AI provider configured, using deterministic fallback");
return null;
}
const evidence: RCAResult["evidence"] = [];
const online = metrics?.metrics?.online;
const catalogId = catalogIdOverride || metrics?.catalogId as string | undefined;
const catalogLabel = catalogLabelOf(catalogId);
// --- Phase 1: AI 规划诊断命令 ---
const phase1User = [
`## 设备型号(关键!请严格按此选命令)`,
`- catalogId: ${catalogId || "unknown"}`,
`- 设备类型: ${catalogLabel}`,
`- 是否在线: ${online}`,
``,
`## 告警信息`,
`- 设备: ${alert.device_name || alert.device_id} (id=${alert.device_id})`,
`- 规则: ${alert.rule_name} (${alert.rule_id})`,
`- 级别: ${alert.level}`,
`- 指标: ${alert.metric} = ${alert.metric_value} (阈值 ${alert.threshold})`,
`- 触发时间: ${new Date(alert.triggered_at).toLocaleString("zh-CN")}`,
`- 消息: ${alert.message}`,
``,
`## 最新巡检 metrics`,
metrics?.metrics ? JSON.stringify(metrics.metrics, null, 2) : "(暂无数据)",
``,
`请输出需要执行的诊断命令列表(JSON)。`,
].join("\n");
const phase1Raw = await aiChatCompletion(provider, phase1System(), [{ role: "user", content: phase1User }]);
if (!phase1Raw) return null;
let plan: { commands: Array<{ command: string; purpose: string }> } | null = null;
try {
const parsed = parseFirstJSON(phase1Raw);
if (parsed && Array.isArray(parsed.commands)) plan = parsed;
} catch (_) {}
if (!plan) return null; // AI 没输出有效计划
// --- 构建 hostConfig(从 vault 解密凭据,独立 SSH)---
const hostConfig = await buildHostConfig(alert.device_id);
console.log("[RCA] buildHostConfig result:", hostConfig ? `hostname=${hostConfig.hostname} user=${hostConfig.username}` : "null");
if (!hostConfig) {
evidence.push({
tool: "inspectionRunSingleCommand",
summary: "无法执行命令(vault 中找不到设备凭据)",
keyFindings: ["提示:请先在 Vault 中配置该设备的主机和凭据"],
});
}
// --- Execute plan via independent SSH(独立建 SSH,不依赖已有终端)---
const executedCmds: Array<{ command: string; output: string; ok: boolean; purpose?: string }> = [];
for (const c of plan.commands) {
if (!hostConfig) {
executedCmds.push({
command: c.command,
output: "skipped — 无 hostConfig",
ok: false,
purpose: c.purpose,
});
continue;
}
const start = performance.now();
try {
console.log("[RCA] exec cmd:", c.command, "on", hostConfig.hostname);
// 优先走 preload inspection 命名空间,兜底直接走 ipcRenderer.invoke
const b = bridge();
let resp: any = null;
if (b.inspection?.runSingleCommand) {
resp = await b.inspection.runSingleCommand({
hostConfig,
command: c.command,
timeoutMs: 15000,
});
} else {
// 兜底:直接调 ipcRenderer(preload 没注册时可能走这里)
const w = window as any;
if (w.netcatty?.ipcRenderer?.invoke) {
resp = await w.netcatty.ipcRenderer.invoke("inspection:runSingleCommand", {
hostConfig,
command: c.command,
timeoutMs: 15000,
});
} else {
throw new Error("inspection.runSingleCommand 不可用");
}
}
const rawOut = (resp?.stdout || "") + (resp?.stderr ? "\n--- stderr ---\n" + resp.stderr : "");
const outStr = _stripVendorBanner(rawOut).slice(0, 5000);
const rawFirst200 = rawOut.slice(0, 200).replace(/\n/g, "\\n");
const cleanFirst200 = outStr.slice(0, 200).replace(/\n/g, "\\n");
console.log("[RCA] cmd result:", resp);
if (rawFirst200 !== cleanFirst200) {
console.log("[RCA] vendor banner stripped:", { raw: rawFirst200, clean: cleanFirst200 });
}
executedCmds.push({
command: c.command,
output: outStr,
ok: true,
purpose: c.purpose,
});
invocations.push({
tool: "inspectionRunCommand",
ok: true,
durationMs: Math.round(performance.now() - start),
});
} catch (e: any) {
executedCmds.push({
command: c.command,
output: e?.message || String(e),
ok: false,
purpose: c.purpose,
});
invocations.push({
tool: "inspectionRunCommand",
ok: false,
durationMs: Math.round(performance.now() - start),
});
}
}
// --- Phase 2: AI 综合 RCA ---
const phase2User = [
`## 设备信息`,
`- catalogId: ${catalogId || "unknown"} (${catalogLabel})`,
`- 是否在线: ${online}`,
``,
`## 告警信息`,
`- 设备: ${alert.device_name || alert.device_id}`,
`- 规则: ${alert.rule_name} (${alert.rule_id})`,
`- 指标: ${alert.metric} = ${alert.metric_value} (阈值 ${alert.threshold})`,
`- 级别: ${alert.level}`,
``,
`## 最新巡检 metrics`,
metrics?.metrics ? JSON.stringify(metrics.metrics, null, 2) : "(暂无数据)",
``,
`## 诊断命令执行结果`,
...executedCmds.map(
(c) =>
`### ${c.command}${c.ok ? "" : " ❌ failed"}\n` +
`用途: ${c.purpose || "-"}\n` +
`\`\`\`\n${c.output.slice(0, 2500)}\n\`\`\``,
),
``,
`请综合以上证据分析根因并输出 JSON。`,
].join("\n");
const phase2Raw = await aiChatCompletion(provider, phase2System(), [{ role: "user", content: phase2User }]);
if (!phase2Raw) return null;
const rca = parseFinalRCA(phase2Raw, alert, evidence);
if (!rca) return null;
// Attach command outputs to evidence
for (const c of executedCmds) {
evidence.push({
tool: c.command,
summary: c.purpose || "诊断命令",
keyFindings: [c.output.slice(0, 500)],
});
}
rca.evidence = evidence;
return {
success: true,
source: "ai",
rca,
rawText: phase2Raw,
durationMs: 0,
toolInvocations: invocations,
};
}
function parseFirstJSON(text: string): any {
let s = text.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/i, "");
try { return JSON.parse(s); } catch (_) {}
const m = s.match(/\{[\s\S]*\}/);
if (m) { try { return JSON.parse(m[0]); } catch (_) {} }
return null;
}
function parseFinalRCA(text: string, alert: AlertRecord, evidence: RCAResult["evidence"]): RCAResult | null {
const parsed = parseFirstJSON(text);
if (!parsed || typeof parsed !== "object") return null;
if (!parsed.rootCauses?.length && !parsed.summary) return null;
return {
alertId: alert.id,
deviceId: alert.device_id,
deviceName: alert.device_name || alert.device_id,
confidence: Math.max(0, Math.min(100, Number(parsed.confidence) || 0)),
rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses.map(String) : [],
evidence,
recommendations: Array.isArray(parsed.recommendations)
? parsed.recommendations.map((r: any) => ({
priority: ["immediate", "short_term", "long_term"].includes(r?.priority) ? r.priority : "short_term",
action: String(r?.action ?? ""),
command: r?.command ? String(r.command) : undefined,
risk: ["low", "medium", "high"].includes(r?.risk) ? r.risk : "medium",
note: r?.note ? String(r.note) : undefined,
}))
: [],
summary: String(parsed.summary ?? ""),
};
}
// ---------- Deterministic ----------
async function runFallbackPath(
alert: AlertRecord,
metrics: any,
invocations: RunRCAResult["toolInvocations"],
): Promise<RCAResult> {
const evidence: RCAResult["evidence"] = [];
const commands = pickFallbackCmds(alert, metrics?.catalogId as string | undefined);
for (const cmd of commands) {
const start = performance.now();
try {
const resp = (await rpc("netcatty/inspectionRunCommand", {
deviceId: alert.device_id,
command: cmd,
timeoutMs: 15000,
})) as any;
const outStr = (resp?.stdout || "") + (resp?.stderr ? "\n--- stderr ---\n" + resp.stderr : "");
evidence.push({ tool: cmd, summary: "诊断命令输出", keyFindings: [outStr.slice(0, 2000)] });
invocations.push({ tool: "inspectionRunCommand", ok: true, durationMs: Math.round(performance.now() - start) });
} catch (e: any) {
evidence.push({ tool: cmd, summary: "命令执行失败", keyFindings: [e?.message || ""] });
invocations.push({ tool: "inspectionRunCommand", ok: false, durationMs: Math.round(performance.now() - start) });
}
}
const offline = alert.rule_id === "rule_offline";
const cpu = alert.metric === "cpu_usage";
const mem = alert.metric === "memory_usage";
const disk = alert.metric === "disk_usage";
const hostLabel = alert.device_name || alert.device_id;
const rootCauses: string[] = [];
const recs: RCAResult["recommendations"] = [];
let confidence = 40;
if (offline) {
rootCauses.push("设备离线或 SSH 连接异常");
confidence = 55;
recs.push({ priority: "immediate", action: "检查网络连通性", command: `ping -c 3 ${hostLabel}`, risk: "low" });
recs.push({ priority: "immediate", action: "确认 SSH 服务和端口", risk: "low" });
}
if (cpu) {
rootCauses.push(`CPU ${alert.metric_value}% 超过阈值 ${alert.threshold}% — 热点进程或 I/O wait`);
confidence = 60;
recs.push({ priority: "immediate", action: "定位 CPU 热点进程", command: "top -bn1 | head -20 && ps aux --sort=-%cpu | head -10", risk: "low" });
}
if (mem) {
rootCauses.push(`内存 ${alert.metric_value}% — 内存泄漏或突发负载`);
confidence = 60;
recs.push({ priority: "immediate", action: "检查 top 内存进程", command: "free -m && ps aux --sort=-%mem | head -10", risk: "low" });
}
if (disk) {
rootCauses.push("磁盘使用率超限 — 日志堆积或大文件残留");
confidence = 60;
recs.push({ priority: "immediate", action: "定位大文件并清理", command: "df -h && du -sh /var/log/* /tmp/* | sort -rh | head -20", risk: "medium", note: "删除前确认不影响日志轮转" });
}
if (!rootCauses.length) {
rootCauses.push("告警触发但信息不足,需要人工排查");
confidence = 25;
recs.push({ priority: "short_term", action: "人工登录设备检查", risk: "low" });
}
return {
alertId: alert.id,
deviceId: alert.device_id,
deviceName: hostLabel,
confidence,
rootCauses,
evidence,
recommendations: recs,
summary: `${hostLabel} 触发 ${alert.level} 级告警(${alert.rule_name})。根因: ${rootCauses[0]}。置信度 ${confidence}%(确定性分析,无 AI 参与)。`,
};
}
export function parseRCAResult(raw: string, alert: AlertRecord): RCAResult | null {
return parseFinalRCA(raw, alert, []);
}