938 lines
38 KiB
TypeScript
938 lines
38 KiB
TypeScript
|
/**
|
|||
|
|
* RCA (Root Cause Analysis) runner — v2 AI-driven.
|
|||
|
|
*
|
|||
|
|
* Two-phase design (no AI SDK tool-loop required):
|
|||
|
|
*
|
|||
|
|
* Phase 1 — Plan:
|
|||
|
|
* AI receives alert + context + system prompt (diagnosis workflow).
|
|||
|
|
* Outputs a JSON plan: which diagnosis commands to run on which device.
|
|||
|
|
*
|
|||
|
|
* Execute — We run the plan:
|
|||
|
|
* For each command → aiCapability("netcatty/inspectionRunCommand", { deviceId, command })
|
|||
|
|
* Collect stdout into evidence bundle.
|
|||
|
|
*
|
|||
|
|
* Phase 2 — Synthesize:
|
|||
|
|
* AI receives alert + metrics + ALL command outputs.
|
|||
|
|
* Outputs final RCAResult JSON (root causes + recommendations).
|
|||
|
|
*
|
|||
|
|
* Deterministic fallback — if any phase fails, we use local heuristics
|
|||
|
|
* (inspection.runCommand still runs real SSH commands).
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
import type { AlertRecord } from "../../inspection/inspectionApi";
|
|||
|
|
import {
|
|||
|
|
STORAGE_KEY_HOSTS,
|
|||
|
|
STORAGE_KEY_KEYS,
|
|||
|
|
} from "../../config/storageKeys";
|
|||
|
|
import {
|
|||
|
|
decryptHosts,
|
|||
|
|
decryptKeys,
|
|||
|
|
} from "../../persistence/secureFieldAdapter";
|
|||
|
|
|
|||
|
|
export interface RunRCAOptions {
|
|||
|
|
forceDeterministic?: boolean;
|
|||
|
|
onTool?: (name: string, ok: boolean, summary: string) => void;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export interface RCAResult {
|
|||
|
|
alertId: string;
|
|||
|
|
deviceId: string;
|
|||
|
|
deviceName: string;
|
|||
|
|
confidence: number;
|
|||
|
|
rootCauses: string[];
|
|||
|
|
evidence: Array<{
|
|||
|
|
tool: string;
|
|||
|
|
summary: string;
|
|||
|
|
keyFindings: string[];
|
|||
|
|
}>;
|
|||
|
|
recommendations: Array<{
|
|||
|
|
priority: "immediate" | "short_term" | "long_term";
|
|||
|
|
action: string;
|
|||
|
|
command?: string;
|
|||
|
|
risk: "low" | "medium" | "high";
|
|||
|
|
note?: string;
|
|||
|
|
}>;
|
|||
|
|
summary: string;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export interface RunRCAResult {
|
|||
|
|
success: boolean;
|
|||
|
|
rca?: RCAResult;
|
|||
|
|
rawText?: string;
|
|||
|
|
source: "ai" | "fallback";
|
|||
|
|
error?: string;
|
|||
|
|
durationMs: number;
|
|||
|
|
toolInvocations: Array<{ tool: string; ok: boolean; durationMs: number }>;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- Bridge (renderer preload) ----------
|
|||
|
|
|
|||
|
|
type Bridge = {
|
|||
|
|
aiCapability?: (
|
|||
|
|
rpcMethod: string,
|
|||
|
|
params: Record<string, unknown>,
|
|||
|
|
chatSessionId?: string,
|
|||
|
|
) => Promise<unknown>;
|
|||
|
|
aiFetch?: (
|
|||
|
|
url: string, method: string, headers: Record<string, string>,
|
|||
|
|
body?: string, providerId?: string, skipHostCheck?: boolean,
|
|||
|
|
followRedirects?: boolean, skipTLSVerify?: boolean, timeoutMs?: number,
|
|||
|
|
) => Promise<{ ok: boolean; status: number; data: string; error?: string }>;
|
|||
|
|
aiAllowlistAddHost?: (baseURL: string) => Promise<unknown>;
|
|||
|
|
aiSyncProviders?: (providers: unknown[]) => Promise<unknown>;
|
|||
|
|
inspection?: {
|
|||
|
|
runSingleCommand?: (payload: {
|
|||
|
|
hostConfig: HostConfig;
|
|||
|
|
command: string;
|
|||
|
|
timeoutMs?: number;
|
|||
|
|
}) => Promise<any>;
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
function bridge(): Bridge {
|
|||
|
|
return (window as unknown as { netcatty?: Bridge }).netcatty ?? ({} as Bridge);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function rpc<T>(method: string, params: Record<string, unknown>): Promise<T> {
|
|||
|
|
const b = bridge();
|
|||
|
|
if (!b.aiCapability) throw new Error("aiCapability bridge unavailable");
|
|||
|
|
const result = await b.aiCapability(method, params);
|
|||
|
|
// aiCapability returns handler result directly — if ok: false, surface error
|
|||
|
|
if (result && typeof result === "object" && (result as any).ok === false) {
|
|||
|
|
throw new Error((result as any).error || "RPC failed");
|
|||
|
|
}
|
|||
|
|
return result as T;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function safeRpc<T>(method: string, params: Record<string, unknown>): Promise<T | null> {
|
|||
|
|
try { return await rpc<T>(method, params); } catch (_) { return null; }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- AI chat completions via aiFetch ----------
|
|||
|
|
|
|||
|
|
interface ProviderInfo {
|
|||
|
|
providerId: string;
|
|||
|
|
apiKey: string;
|
|||
|
|
baseUrl: string;
|
|||
|
|
model: string;
|
|||
|
|
style?: string;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Storage keys (keep in sync with infrastructure/config/storageKeys.ts)
|
|||
|
|
const AI_PROVIDERS_KEY = "netcatty_ai_providers_v1";
|
|||
|
|
const AI_ACTIVE_PROVIDER_KEY = "netcatty_ai_active_provider_v1";
|
|||
|
|
|
|||
|
|
async function resolveProvider(): Promise<ProviderInfo | null> {
|
|||
|
|
let providers: any[] | null = null;
|
|||
|
|
let activeId: string | null = null;
|
|||
|
|
try {
|
|||
|
|
const raw = localStorage.getItem(AI_PROVIDERS_KEY);
|
|||
|
|
if (raw) providers = JSON.parse(raw);
|
|||
|
|
activeId = localStorage.getItem(AI_ACTIVE_PROVIDER_KEY);
|
|||
|
|
} catch (_) {}
|
|||
|
|
if (!Array.isArray(providers) || providers.length === 0) return null;
|
|||
|
|
|
|||
|
|
// Prefer active provider, fall back to first enabled one
|
|||
|
|
let primary: any = null;
|
|||
|
|
if (activeId) primary = providers.find((p) => p.id === activeId);
|
|||
|
|
if (!primary) primary = providers.find((p) => p.enabled !== false);
|
|||
|
|
if (!primary?.apiKey) return null;
|
|||
|
|
|
|||
|
|
// 同步 providers 到主进程（主进程解密 + host allowlist 都依赖它）
|
|||
|
|
// Catty SDK 正常流程会自动 sync，RCA 跳过了所以这里手动补一次
|
|||
|
|
const b = bridge();
|
|||
|
|
if (b.aiSyncProviders) {
|
|||
|
|
try { await b.aiSyncProviders(providers); } catch (_) {}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const baseUrl = primary.baseURL || "";
|
|||
|
|
const style = primary.style || detectStyle(baseUrl || primary.providerId || "");
|
|||
|
|
const path = style === "anthropic" ? "/v1/messages"
|
|||
|
|
: style === "google" ? "/v1beta/models"
|
|||
|
|
: "/chat/completions";
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
providerId: primary.id || "default",
|
|||
|
|
apiKey: primary.apiKey, // 可能是 enc:v1: 加密 placeholder，主进程 injectApiKeyIntoRequest 会解密替换
|
|||
|
|
baseUrl: (baseUrl.replace(/\/$/, "")) + path,
|
|||
|
|
model: primary.defaultModel || "gpt-4o-mini",
|
|||
|
|
style,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 清洗厂商登录 banner / URL / 版权噪音，让 AI 只看到命令输出 */
|
|||
|
|
function _stripVendorBanner(text: string): string {
|
|||
|
|
if (!text) return text;
|
|||
|
|
let out = text;
|
|||
|
|
// 1. 删所有 http/https URL
|
|||
|
|
out = out.replace(/https?:\/\/[^\s|,，。]+/g, "");
|
|||
|
|
// 2. 删裸域名 www.xxx.com / 管理门户路径
|
|||
|
|
out = out.replace(/\bwww\.[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?/gi, "");
|
|||
|
|
out = out.replace(/\bmanagement\.[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?/gi, "");
|
|||
|
|
// 3. 删残留的 Web 脚本后缀片段（banner 里可能出现裸 .aspx / .do 等）
|
|||
|
|
out = out.replace(/\.(aspx|do|cgi|php|jsp|html?|shtml|asp|cfm|jspx|action)(\/[^\s]*)?\b/gi, "");
|
|||
|
|
// 4. 删整行噪音
|
|||
|
|
out = out
|
|||
|
|
.split("\n")
|
|||
|
|
.filter(line => {
|
|||
|
|
const t = line.trim();
|
|||
|
|
if (!t) return true;
|
|||
|
|
// SSH 登录 banner
|
|||
|
|
if (/Welcome to (Kylin|Ubuntu|CentOS|Red Hat|RedHat|Debian|SUSE|openSUSE|Fedora|AlmaLinux|Rocky Linux)/i.test(t)) return false;
|
|||
|
|
if (/^(Huawei|H3C|Cisco|Ruijie|ZTE|锐捷|中兴|华为)/.test(t) && /(Platform|Software|Operating System|Comware|VRP)/i.test(t)) return false;
|
|||
|
|
// 管理门户 / 版权 / 警告
|
|||
|
|
if (/^\s*\*?\s*Management\s*[:：]/i.test(t)) return false;
|
|||
|
|
if (/^\s*\*?\s*(Copyright|Trademark|All Rights Reserved|For Internal Use|Unauthorized)/i.test(t)) return false;
|
|||
|
|
if (/^\s*\*{5,}\s*$/.test(t)) return false;
|
|||
|
|
// SSH MOTD（登录欢迎信息）
|
|||
|
|
if (/^Last login:\s/i.test(t)) return false;
|
|||
|
|
if (/^You have (new )?mail/i.test(t)) return false;
|
|||
|
|
if (/^No mail$/i.test(t)) return false;
|
|||
|
|
if (/^MOTD(\s|$)/i.test(t)) return false;
|
|||
|
|
if (/^Message of the Day/i.test(t)) return false;
|
|||
|
|
// Kylin / Ubuntu 的欢迎行（如 "Welcome to Kylin V10 SP1..." 已在上面，这里兜底）
|
|||
|
|
if (/^\s*\*?\s*(Welcome|欢迎)/i.test(t) && /(Kylin|Ubuntu|CentOS|Linux|操作系统)/i.test(t)) return false;
|
|||
|
|
// 麒麟 OS 的软件协议 / 许可提示
|
|||
|
|
if (/^\s*\*?\s*(License|许可|授权|Service Pack|SP\d+)/i.test(t) && /(Kylin|麒麟)/i.test(t)) return false;
|
|||
|
|
return true;
|
|||
|
|
})
|
|||
|
|
.join("\n");
|
|||
|
|
// 5. 多空行压缩
|
|||
|
|
out = out.replace(/\n{3,}/g, "\n\n").trim();
|
|||
|
|
return out;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function detectStyle(baseUrlOrId: string): string {
|
|||
|
|
const lower = baseUrlOrId.toLowerCase();
|
|||
|
|
if (/anthropic|claude/.test(lower)) return "anthropic";
|
|||
|
|
if (/google|gemini/.test(lower)) return "google";
|
|||
|
|
return "openai";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const CATALOG_LABELS: Record<string, string> = {
|
|||
|
|
linux: "Linux 服务器",
|
|||
|
|
linux_server: "Linux 服务器",
|
|||
|
|
windows: "Windows 服务器",
|
|||
|
|
huawei_h3c_switch: "华为/H3C 交换机",
|
|||
|
|
cisco_switch: "Cisco 交换机",
|
|||
|
|
huawei_firewall: "华为防火墙",
|
|||
|
|
fortinet_firewall: "飞塔防火墙",
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
function catalogLabelOf(catalogId?: string): string {
|
|||
|
|
if (!catalogId) return "未知（unknown）";
|
|||
|
|
return CATALOG_LABELS[catalogId] || catalogId;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function aiChatCompletion(
|
|||
|
|
provider: ProviderInfo,
|
|||
|
|
system: string,
|
|||
|
|
messages: Array<{ role: "user" | "assistant"; content: string }>,
|
|||
|
|
): Promise<string> {
|
|||
|
|
const b = bridge();
|
|||
|
|
if (!b.aiFetch) throw new Error("aiFetch bridge unavailable");
|
|||
|
|
|
|||
|
|
// 预注册 host 到主进程 allowlist（独立 RCA 路径不走 Catty SDK，没同步过 provider host）
|
|||
|
|
// 幂等调用 — host 已存在时只是刷新 TTL
|
|||
|
|
if ((b as any).aiAllowlistAddHost) {
|
|||
|
|
try { await (b as any).aiAllowlistAddHost(provider.baseUrl); } catch (_) {}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const body = JSON.stringify({
|
|||
|
|
model: provider.model,
|
|||
|
|
messages: [
|
|||
|
|
{ role: "system", content: system },
|
|||
|
|
...messages,
|
|||
|
|
],
|
|||
|
|
temperature: 0,
|
|||
|
|
stream: false,
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
const headers: Record<string, string> = {
|
|||
|
|
"Content-Type": "application/json",
|
|||
|
|
// 用 placeholder — 主进程 injectApiKeyIntoRequest 会用 providerId 解密注入真实 key
|
|||
|
|
Authorization: `Bearer __IPC_SECURED__`,
|
|||
|
|
};
|
|||
|
|
if (provider.style === "anthropic") {
|
|||
|
|
headers["anthropic-version"] = "2023-06-01";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// RCA 分析也是多轮思考 + 大输出，给 120s timeout
|
|||
|
|
const resp = await b.aiFetch(
|
|||
|
|
provider.baseUrl, "POST", headers, body, provider.providerId,
|
|||
|
|
/*skipHostCheck*/ true, /*followRedirects*/ false, /*skipTLSVerify*/ false,
|
|||
|
|
/*timeoutMs*/ 1200000,
|
|||
|
|
);
|
|||
|
|
if (!resp.ok) throw new Error(`AI ${resp.status}: ${resp.data?.slice(0, 200) || resp.error}`);
|
|||
|
|
return parseAiChat(resp.data || "{}", provider.style);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function parseAiChat(data: string, style?: string): string {
|
|||
|
|
const parsed = JSON.parse(data || "{}");
|
|||
|
|
if (style === "anthropic") return parsed.content?.[0]?.text ?? "";
|
|||
|
|
if (style === "google") return parsed.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
|
|||
|
|
return parsed.choices?.[0]?.message?.content ?? "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- System prompts ----------
|
|||
|
|
// 内置默认值 —— 存储在 STORAGE_KEY_RCA_PROMPTS 里的自定义 prompt 优先使用
|
|||
|
|
|
|||
|
|
export const DEFAULT_PHASE1_SYSTEM = [
|
|||
|
|
`你是 NetMesh 运维平台的 AI 诊断规划专家。`,
|
|||
|
|
`平台会在设备上执行你规划的每条命令，收集 stdout 返回给你。`,
|
|||
|
|
`如果设备离线（online=false），不要跑任何需要 SSH 的命令 — 改成先确认网络可达性。`,
|
|||
|
|
``,
|
|||
|
|
`## 设备 Catalog 与命令族（必须严格匹配）`,
|
|||
|
|
`| catalogId | 设备类型 | Shell | 实际巡检命令（系统已在用） | 推荐诊断命令 |`,
|
|||
|
|
`|---|---|---|---|---|`,
|
|||
|
|
`| linux | Linux 服务器 | bash/sh | top -bn1 / mpstat 1 1 / free -k / cat /proc/meminfo / df -h | top -bn1 \| head -20, vmstat 1 2, free -m, cat /proc/meminfo \| head -20, df -h, ps aux --sort=-%cpu \| head -10, iostat -x 1 1 2>/dev/null \|\| echo no_iostat, lsof -i, journalctl -xe --no-pager \| tail -50, dmesg \| tail -50 |`,
|
|||
|
|
`| windows | Windows 服务器 | PowerShell | Get-WmiObject Win32_Processor / Win32_OperatingSystem / Win32_LogicalDisk / Get-Service | Get-Process \| Sort-Object CPU -Descending \| Select-Object -First 10, Get-CimInstance Win32_OperatingSystem, Get-CimInstance Win32_LogicalDisk, Get-Service \| Where-Object {$_.Status -ne 'Running'}, Get-EventLog -LogName System -Newest 50 |`,
|
|||
|
|
`| huawei_h3c_switch | 华为/H3C 交换机 | HUAWEI CLI | display cpu-usage / display memory / display interface brief / display power / display fan | **禁止 display cpu-usage（有刷新动画抓不到完整输出）**；可用命令: display version, display device, display memory, display power, display fan, display logbuffer, display stp brief, display vrrp, display vlan brief, display arp |`,
|
|||
|
|
`| cisco_switch | Cisco 交换机 | IOS CLI | 复用华为交换机巡检（暂无专用 adapter） | show version, show processes cpu sorted, show memory summary, show interfaces brief, show interfaces counters errors, show inventory, show log tail 50, show vlan brief, show spanning-tree summary |`,
|
|||
|
|
`| huawei_firewall | 华为防火墙 | HUAWEI CLI | 复用华为交换机巡检（暂无专用 adapter） | display version, display cpu-usage, display memory, display firewall session table, display current-configuration, display security-policy count, display server-protect statistics |`,
|
|||
|
|
`| fortinet_firewall | 飞塔防火墙 | FortiOS CLI | 复用华为交换机巡检（暂无专用 adapter） | get system status, get system performance status, get system performance cpu, get system performance memory, get firewall session list, get config route static, diagnose debug flow stat |`,
|
|||
|
|
``,
|
|||
|
|
`## 输出格式（严格 JSON，不要 markdown 代码块之外的文字）`,
|
|||
|
|
`{`,
|
|||
|
|
` "commands": [`,
|
|||
|
|
` { "command": "display cpu-usage", "purpose": "查看设备 CPU 使用率" }`,
|
|||
|
|
` ],`,
|
|||
|
|
` "reasoning": "一句话说明为什么选这些命令"`,
|
|||
|
|
`}`,
|
|||
|
|
``,
|
|||
|
|
`## 命令规划规则`,
|
|||
|
|
`- **命令必须严格从上面"推荐诊断命令"列中选择**，不得臆造未知命令（如 display process、display memory-usage）`,
|
|||
|
|
`- **严禁输出任何外部 URL / 厂商官网链接 / 技术文档链接**（如 management.h3c.com、www.kylinos.cn、cisco.com/wiki、forum.huawei 等） —— 你的运行环境在内网，外网链接无法访问，输出它们没有任何意义`,
|
|||
|
|
`- **严禁引用厂商文档名称和章节号**（如"参见 H3C S5700 V200R010 配置指南"），设备 firmware / 型号信息必须从诊断命令输出中提取`,
|
|||
|
|
`- **严禁推荐访问外网工具**（如 curl/wget 拉远程脚本、apt/yum 从公网仓库安装）`,
|
|||
|
|
`- **禁止选择语义重复的命令**（如 display memory 和 display memory-usage 只需选一条）`,
|
|||
|
|
`- **禁止选择持续刷新/交互式命令** — display cpu-usage 有动态刷新动画，无法完整抓取输出。请改用 display device（含 CPU 模块状态）或 display power`,
|
|||
|
|
`- **关闭分页的前置命令**（让 display logbuffer 完整输出）：华为/H3C 用 "screen-length 0 temporary"（用户视图下可用），不要用 "terminal length 0"（那是 system-view 下的）。独立作为 commands 数组里的第一条`,
|
|||
|
|
`- **display logbuffer** 建议用 "display logbuffer level warning" 或 "display logbuffer size 512" 缩小输出范围，避免 10000+ 行日志`,
|
|||
|
|
`- display version 是只读静态信息，建议放在 plan 里至少一次（用来识别设备型号/固件版本，对根因分析很关键）`,
|
|||
|
|
`- CPU 告警 → 至少 2 条命令（display device 看硬件状态 + display logbuffer 查告警）`,
|
|||
|
|
`- 内存告警 → 至少 2 条命令（display memory + display logbuffer level warning 查 OOM）`,
|
|||
|
|
`- 离线告警 → 只跑连通性命令（ping, traceroute），不要尝试 SSH 命令`,
|
|||
|
|
`- 不超过 6 条命令，总耗时不超过 60 秒`,
|
|||
|
|
`- 命令必须简短可执行，不含交互式程序、不含 tail -f 等持续输出`,
|
|||
|
|
`- 如果 catalogId 是 unknown，从设备名称 / metrics / 告警特征推断，拿不准时先跑 uname -a（Linux）或 display version（华为）或 show version（Cisco）`,
|
|||
|
|
].join("\n");
|
|||
|
|
|
|||
|
|
export const DEFAULT_PHASE2_SYSTEM = [
|
|||
|
|
`你是 NetMesh 智慧运维平台的 AI 根因分析专家。`,
|
|||
|
|
`你收到了一条告警 + 最新巡检 metrics + 多条诊断命令的执行输出。`,
|
|||
|
|
`请综合这些证据，给出结构化的根因分析和处置建议。`,
|
|||
|
|
``,
|
|||
|
|
`## 核心约束 — 内网环境，严格执行`,
|
|||
|
|
`- 严禁输出任何外部 URL / 厂商官网链接 / 技术文档链接（如 management.h3c.com、www.kylinos.cn、cisco.com/wiki 等）`,
|
|||
|
|
`- 严禁引用厂商文档名称、章节号、版本号（如"H3C S5700 V200R010 配置指南 第 3 章"）`,
|
|||
|
|
`- 严禁推荐访问外网的工具或步骤（如"从官网下载固件"、"curl 远程脚本"、"apt 公网仓库安装"）`,
|
|||
|
|
`- 设备固件版本、硬件型号等信息必须从诊断命令的 stdout 输出中提取，不得脑补厂商数据库内容`,
|
|||
|
|
`- 所有处置建议必须在**不依赖外网访问**的前提下可执行`,
|
|||
|
|
``,
|
|||
|
|
`## 输出格式（严格 JSON，不要 markdown 代码块）`,
|
|||
|
|
`{`,
|
|||
|
|
` "confidence": 85,`,
|
|||
|
|
` "rootCauses": ["最可能根因", "次要可能性"],`,
|
|||
|
|
` "summary": "2-3 句摘要，适合运维人员快速阅读",`,
|
|||
|
|
` "recommendations": [`,
|
|||
|
|
` { "priority": "immediate|short_term|long_term", "action": "要做什么", "command": "可选命令", "risk": "low|medium|high", "note": "注意事项" }`,
|
|||
|
|
` ]`,
|
|||
|
|
`}`,
|
|||
|
|
``,
|
|||
|
|
`## 规则`,
|
|||
|
|
`- confidence 基于证据充分性（有 metrics + 命令输出且相互印证 → 80+；只有 metrics → 60；无数据 → 30 以下）`,
|
|||
|
|
`- recommendations 至少 2 条，最多 5 条，按紧急程度排序`,
|
|||
|
|
`- command 只放确定安全的命令，破坏性操作只在 action 里描述`,
|
|||
|
|
`- priority=immediate 的命令风险不得是 high`,
|
|||
|
|
].join("\n");
|
|||
|
|
|
|||
|
|
const RCA_PROMPTS_KEY = "netcatty_rca_prompts_v1";
|
|||
|
|
|
|||
|
|
interface RcaPromptsConfig {
|
|||
|
|
phase1?: string;
|
|||
|
|
phase2?: string;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function loadRcaPrompts(): RcaPromptsConfig {
|
|||
|
|
try {
|
|||
|
|
const raw = localStorage.getItem(RCA_PROMPTS_KEY);
|
|||
|
|
if (raw) return JSON.parse(raw);
|
|||
|
|
} catch (_) {}
|
|||
|
|
return {};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function phase1System(): string {
|
|||
|
|
const c = loadRcaPrompts();
|
|||
|
|
return c.phase1?.trim() ? c.phase1 : DEFAULT_PHASE1_SYSTEM;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function phase2System(): string {
|
|||
|
|
const c = loadRcaPrompts();
|
|||
|
|
return c.phase2?.trim() ? c.phase2 : DEFAULT_PHASE2_SYSTEM;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 设备类型自动探测 ----------
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 未知 catalogId 时，发 3 条不同命令族的探测命令，根据哪条成功自动识别设备类型。
|
|||
|
|
* 探测顺序：
|
|||
|
|
* 1. display memory → 华为/H3C 交换机/防火墙
|
|||
|
|
* 2. show memory → Cisco IOS 交换机/防火墙
|
|||
|
|
* 3. free -k → Linux bash
|
|||
|
|
* 全部失败 → catalogId 继续 undefined，AI 会用 unknown 兜底
|
|||
|
|
*/
|
|||
|
|
async function autoDetectCatalog(hostConfig: HostConfig): Promise<string | undefined> {
|
|||
|
|
const probes: Array<{ cmd: string; catalogId: string; negative?: RegExp }> = [
|
|||
|
|
{ cmd: "display memory", catalogId: "huawei_h3c_switch", negative: /未找到命令|not found|unknown command|unrecognized command|invalid input/i },
|
|||
|
|
{ cmd: "show memory", catalogId: "cisco_switch", negative: /not found|invalid input|unknown command/i },
|
|||
|
|
{ cmd: "free -k", catalogId: "linux", negative: /command not found|未找到命令|not found/i },
|
|||
|
|
];
|
|||
|
|
|
|||
|
|
for (const probe of probes) {
|
|||
|
|
try {
|
|||
|
|
const resp = await bridge().inspection!.runSingleCommand!(
|
|||
|
|
{ hostConfig, command: probe.cmd, timeoutMs: 8000 },
|
|||
|
|
);
|
|||
|
|
const result = resp as any;
|
|||
|
|
const stdout = (result?.stdout || "") as string;
|
|||
|
|
const stderr = (result?.stderr || "") as string;
|
|||
|
|
|
|||
|
|
// 1. SSH 层就失败了（离线/认证）→ 跳过所有探测
|
|||
|
|
if (!result?.ok && /SSH|timeout|EAI_|ENOTFOUND|ECONNREFUSED|ECONNRESET/i.test(stderr || result?.error || "")) {
|
|||
|
|
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ SSH 连接问题，跳过:", stderr || result?.error);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 2. 命令不存在（中英文都匹配）→ 继续下一个 probe
|
|||
|
|
const allOutput = stdout + "\n" + stderr;
|
|||
|
|
if (probe.negative && probe.negative.test(allOutput)) {
|
|||
|
|
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ 命令不存在:",
|
|||
|
|
allOutput.slice(0, 80).replace(/[\r\n]+/g, " | "));
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 3. ok=true 且输出有实质内容（>30 字符，不是只有 banner）→ 确认 catalog
|
|||
|
|
// Linux free -k 输出以 "Mem:" "Swap:" 开头
|
|||
|
|
// 交换机 display memory 输出包含 "Total Memory" 或 "Memory Size"
|
|||
|
|
// 如果输出里有明显的 "未找到命令" "command not found" 就 continue 下一个
|
|||
|
|
const okAndHasContent = result?.ok === true
|
|||
|
|
&& stdout.length > 30
|
|||
|
|
&& !probe.negative.test(allOutput);
|
|||
|
|
|
|||
|
|
if (okAndHasContent) {
|
|||
|
|
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ success, catalogId =", probe.catalogId,
|
|||
|
|
"stdout sample:", stdout.slice(0, 150).replace(/[\r\n]+/g, " | "));
|
|||
|
|
return probe.catalogId;
|
|||
|
|
}
|
|||
|
|
} catch (e) {
|
|||
|
|
console.log("[RCA] autoDetectCatalog:", probe.cmd, "→ error:", (e as Error)?.message);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
console.log("[RCA] autoDetectCatalog: all probes failed, catalogId stays undefined");
|
|||
|
|
return undefined;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- Fallback 命令（无 AI / AI 失败时用） ----------
|
|||
|
|
|
|||
|
|
function pickFallbackCmds(alert: AlertRecord, catalogId?: string): string[] {
|
|||
|
|
const offline = alert.rule_id === "rule_offline";
|
|||
|
|
const cpu = alert.metric === "cpu_usage";
|
|||
|
|
const mem = alert.metric === "memory_usage";
|
|||
|
|
const disk = alert.metric === "disk_usage";
|
|||
|
|
const isSwitch = /switch|huawei|h3c|cisco/i.test(catalogId || "");
|
|||
|
|
|
|||
|
|
if (offline) return ["ping -c 3 127.0.0.1", "ssh -V 2>&1"];
|
|||
|
|
if (isSwitch) {
|
|||
|
|
if (cpu) return ["display cpu-usage"];
|
|||
|
|
if (mem) return ["display memory"];
|
|||
|
|
if (disk) return ["display device"];
|
|||
|
|
return ["display version"];
|
|||
|
|
}
|
|||
|
|
if (cpu) return ["top -bn1 | head -20", "vmstat 1 2"];
|
|||
|
|
if (mem) return ["free -m", "cat /proc/meminfo | head -20"];
|
|||
|
|
if (disk) return ["df -h", "iostat -x 1 1 2>/dev/null || echo no_iostat"];
|
|||
|
|
return ["uname -a && uptime && ps aux --sort=-%cpu | head -5"];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 构建 hostConfig（renderer 侧解密 vault hosts + keys）----------
|
|||
|
|
|
|||
|
|
const DEVICE_EXTENSION_KEY = "netcatty_inspection_device_extension_v1";
|
|||
|
|
const INSPECTION_RESULTS_KEY = "netcatty_inspection_inspection_results_v1";
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 从 localStorage 读 inspection device extension 拿 catalog_id（renderer 直接读，不走主进程 handler）。
|
|||
|
|
* 这是 catalogId 的终极兜底 —— 因为 inspectionAgentHandlers 用的 storage adapter 和巡检调度器不同，
|
|||
|
|
* 会读不到 extension，这里 renderer 直接从 localStorage 拿最可靠。
|
|||
|
|
*/
|
|||
|
|
function localCatalogIdOf(deviceId: string): string | undefined {
|
|||
|
|
try {
|
|||
|
|
// 1. 最新巡检结果里的 catalog_id（改之前存的老 result 没有，改之后新巡检会存）
|
|||
|
|
const resultsRaw = localStorage.getItem(INSPECTION_RESULTS_KEY);
|
|||
|
|
console.log("[RCA] localStorage.getItem", INSPECTION_RESULTS_KEY, "→", resultsRaw ? resultsRaw.slice(0, 200) + "..." : "null");
|
|||
|
|
if (resultsRaw) {
|
|||
|
|
const all = JSON.parse(resultsRaw);
|
|||
|
|
if (Array.isArray(all)) {
|
|||
|
|
// 从新到旧找这个 deviceId 的第一条有 catalog_id 的
|
|||
|
|
for (let i = all.length - 1; i >= 0; i--) {
|
|||
|
|
const r = all[i];
|
|||
|
|
if (r?.device_id === deviceId && r?.catalog_id) {
|
|||
|
|
console.log("[RCA] found catalogId in inspection_results:", r.catalog_id, "device_id:", r.device_id);
|
|||
|
|
return r.catalog_id;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// 2. device extension（这个始终有，只要用户在巡检中心给设备选过类型）
|
|||
|
|
const extRaw = localStorage.getItem(DEVICE_EXTENSION_KEY);
|
|||
|
|
console.log("[RCA] localStorage.getItem", DEVICE_EXTENSION_KEY, "→", extRaw ? extRaw.slice(0, 300) + "..." : "null");
|
|||
|
|
if (extRaw) {
|
|||
|
|
const all = JSON.parse(extRaw);
|
|||
|
|
console.log("[RCA] device_extension keys:", Object.keys(all || {}));
|
|||
|
|
const ext = all?.[deviceId];
|
|||
|
|
console.log("[RCA] device_extension[", deviceId, "] →", ext);
|
|||
|
|
if (ext?.catalog_id) {
|
|||
|
|
console.log("[RCA] found catalogId in device_extension:", ext.catalog_id);
|
|||
|
|
return ext.catalog_id;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// 3. 遍历 device_extension 所有 key 找 host_id 匹配（用户可能是用 host_id 存的不是 device_id）
|
|||
|
|
if (extRaw) {
|
|||
|
|
const all = JSON.parse(extRaw);
|
|||
|
|
for (const [key, val] of Object.entries(all || {})) {
|
|||
|
|
const v: any = val;
|
|||
|
|
if (v?.host_id === deviceId || key === deviceId) {
|
|||
|
|
console.log("[RCA] matched ext by key/host_id:", key, v?.catalog_id);
|
|||
|
|
if (v?.catalog_id) return v.catalog_id;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (e) {
|
|||
|
|
console.warn("[RCA] localCatalogIdOf error:", e?.message);
|
|||
|
|
}
|
|||
|
|
console.log("[RCA] localCatalogIdOf returned undefined for", deviceId);
|
|||
|
|
return undefined;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
interface HostConfig {
|
|||
|
|
hostname: string;
|
|||
|
|
port: number;
|
|||
|
|
username: string;
|
|||
|
|
password?: string;
|
|||
|
|
privateKeyContent?: string;
|
|||
|
|
passphrase?: string;
|
|||
|
|
label?: string;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 从 localStorage 读取 vault hosts + keys，解密后构建完整 SSH 连接配置。
|
|||
|
|
* 这是 AI RCA 独立建 SSH 的关键 — 不走已有终端 session，直接从 vault 拿凭据。
|
|||
|
|
*/
|
|||
|
|
async function buildHostConfig(deviceId: string): Promise<HostConfig | null> {
|
|||
|
|
try {
|
|||
|
|
const hostsRaw = localStorage.getItem(STORAGE_KEY_HOSTS);
|
|||
|
|
const keysRaw = localStorage.getItem(STORAGE_KEY_KEYS);
|
|||
|
|
if (!hostsRaw) return null;
|
|||
|
|
|
|||
|
|
const hostsEnc = JSON.parse(hostsRaw);
|
|||
|
|
const keysEnc = keysRaw ? JSON.parse(keysRaw) : [];
|
|||
|
|
|
|||
|
|
const hosts = await decryptHosts(hostsEnc);
|
|||
|
|
const keys = await decryptKeys(keysEnc);
|
|||
|
|
|
|||
|
|
const host = hosts.find((h: any) => h.id === deviceId);
|
|||
|
|
if (!host) return null;
|
|||
|
|
|
|||
|
|
const cfg: HostConfig = {
|
|||
|
|
hostname: host.hostname,
|
|||
|
|
port: host.port || 22,
|
|||
|
|
username: host.username,
|
|||
|
|
label: host.label || host.hostname,
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
if (host.password) cfg.password = host.password;
|
|||
|
|
|
|||
|
|
// 私钥
|
|||
|
|
const identityId = host.identityFileId;
|
|||
|
|
if (identityId) {
|
|||
|
|
const key = keys.find((k: any) => k.id === identityId);
|
|||
|
|
if (key?.privateKey) {
|
|||
|
|
cfg.privateKeyContent = key.privateKey;
|
|||
|
|
if (key.passphrase) cfg.passphrase = key.passphrase;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!cfg.hostname || !cfg.username) return null;
|
|||
|
|
return cfg;
|
|||
|
|
} catch (_) {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- Main entry ----------
|
|||
|
|
|
|||
|
|
export async function runRCA(alert: AlertRecord, opts: RunRCAOptions = {}): Promise<RunRCAResult> {
|
|||
|
|
const t0 = Date.now();
|
|||
|
|
const invocations: RunRCAResult["toolInvocations"] = [];
|
|||
|
|
|
|||
|
|
// Pre-fetch metrics (both AI path and fallback need them)
|
|||
|
|
let metrics: any = null;
|
|||
|
|
let catalogId: string | undefined;
|
|||
|
|
try {
|
|||
|
|
const r = await rpc<any>("netcatty/inspectionHostMetrics", { deviceId: alert.device_id });
|
|||
|
|
metrics = r;
|
|||
|
|
catalogId = (r as any)?.catalogId as string | undefined;
|
|||
|
|
invocations.push({ tool: "inspectionHostMetrics", ok: true, durationMs: 0 });
|
|||
|
|
} catch (_) {}
|
|||
|
|
|
|||
|
|
// Metrics 没有 catalogId（设备离线无巡检数据）→ 从 Vault + device extension 兜底
|
|||
|
|
if (!catalogId) {
|
|||
|
|
try {
|
|||
|
|
const info = await rpc<any>("netcatty/inspectionHostInfo", { deviceId: alert.device_id });
|
|||
|
|
catalogId = (info as any)?.catalogId as string | undefined;
|
|||
|
|
if (metrics && !metrics.catalogId && catalogId) metrics = { ...metrics, catalogId };
|
|||
|
|
invocations.push({ tool: "inspectionHostInfo", ok: true, durationMs: 0 });
|
|||
|
|
} catch (_) {}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 终极兜底：renderer 直接从 localStorage 读 inspection device extension
|
|||
|
|
if (!catalogId) {
|
|||
|
|
catalogId = localCatalogIdOf(alert.device_id);
|
|||
|
|
if (metrics && !metrics.catalogId && catalogId) metrics = { ...metrics, catalogId };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
console.log("[RCA] alert.device_id:", alert.device_id, "| catalogId:", catalogId, "| metrics?.catalogId:", metrics?.catalogId);
|
|||
|
|
|
|||
|
|
// 终极兜底：如果所有上游都没拿到 catalogId，发 3 条探测命令自动识别设备类型
|
|||
|
|
if (!catalogId) {
|
|||
|
|
try {
|
|||
|
|
const hostConfig = await buildHostConfig(alert.device_id);
|
|||
|
|
if (hostConfig) {
|
|||
|
|
catalogId = await autoDetectCatalog(hostConfig);
|
|||
|
|
console.log("[RCA] autoDetectCatalog result:", catalogId);
|
|||
|
|
if (catalogId) {
|
|||
|
|
if (metrics && !metrics.catalogId) metrics = { ...metrics, catalogId };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (_) {}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Try AI path
|
|||
|
|
if (!opts.forceDeterministic) {
|
|||
|
|
const aiResult = await runAIPath(alert, metrics, invocations, catalogId);
|
|||
|
|
if (aiResult) {
|
|||
|
|
aiResult.durationMs = Date.now() - t0;
|
|||
|
|
return aiResult;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Deterministic fallback
|
|||
|
|
const rca = await runFallbackPath(alert, metrics, invocations);
|
|||
|
|
return {
|
|||
|
|
success: true,
|
|||
|
|
source: "fallback",
|
|||
|
|
rca,
|
|||
|
|
durationMs: Date.now() - t0,
|
|||
|
|
toolInvocations: invocations,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- AI two-phase ----------
|
|||
|
|
|
|||
|
|
async function runAIPath(
|
|||
|
|
alert: AlertRecord,
|
|||
|
|
metrics: any,
|
|||
|
|
invocations: RunRCAResult["toolInvocations"],
|
|||
|
|
catalogIdOverride?: string,
|
|||
|
|
): Promise<RunRCAResult | null> {
|
|||
|
|
const provider = await resolveProvider();
|
|||
|
|
if (!provider) {
|
|||
|
|
console.info("[RCA] No AI provider configured, using deterministic fallback");
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const evidence: RCAResult["evidence"] = [];
|
|||
|
|
const online = metrics?.metrics?.online;
|
|||
|
|
const catalogId = catalogIdOverride || metrics?.catalogId as string | undefined;
|
|||
|
|
const catalogLabel = catalogLabelOf(catalogId);
|
|||
|
|
|
|||
|
|
// --- Phase 1: AI 规划诊断命令 ---
|
|||
|
|
const phase1User = [
|
|||
|
|
`## 设备型号（关键！请严格按此选命令）`,
|
|||
|
|
`- catalogId: ${catalogId || "unknown"}`,
|
|||
|
|
`- 设备类型: ${catalogLabel}`,
|
|||
|
|
`- 是否在线: ${online}`,
|
|||
|
|
``,
|
|||
|
|
`## 告警信息`,
|
|||
|
|
`- 设备: ${alert.device_name || alert.device_id} (id=${alert.device_id})`,
|
|||
|
|
`- 规则: ${alert.rule_name} (${alert.rule_id})`,
|
|||
|
|
`- 级别: ${alert.level}`,
|
|||
|
|
`- 指标: ${alert.metric} = ${alert.metric_value} (阈值 ${alert.threshold})`,
|
|||
|
|
`- 触发时间: ${new Date(alert.triggered_at).toLocaleString("zh-CN")}`,
|
|||
|
|
`- 消息: ${alert.message}`,
|
|||
|
|
``,
|
|||
|
|
`## 最新巡检 metrics`,
|
|||
|
|
metrics?.metrics ? JSON.stringify(metrics.metrics, null, 2) : "(暂无数据)",
|
|||
|
|
``,
|
|||
|
|
`请输出需要执行的诊断命令列表（JSON）。`,
|
|||
|
|
].join("\n");
|
|||
|
|
|
|||
|
|
const phase1Raw = await aiChatCompletion(provider, phase1System(), [{ role: "user", content: phase1User }]);
|
|||
|
|
if (!phase1Raw) return null;
|
|||
|
|
|
|||
|
|
let plan: { commands: Array<{ command: string; purpose: string }> } | null = null;
|
|||
|
|
try {
|
|||
|
|
const parsed = parseFirstJSON(phase1Raw);
|
|||
|
|
if (parsed && Array.isArray(parsed.commands)) plan = parsed;
|
|||
|
|
} catch (_) {}
|
|||
|
|
if (!plan) return null; // AI 没输出有效计划
|
|||
|
|
|
|||
|
|
// --- 构建 hostConfig（从 vault 解密凭据，独立 SSH）---
|
|||
|
|
const hostConfig = await buildHostConfig(alert.device_id);
|
|||
|
|
console.log("[RCA] buildHostConfig result:", hostConfig ? `hostname=${hostConfig.hostname} user=${hostConfig.username}` : "null");
|
|||
|
|
if (!hostConfig) {
|
|||
|
|
evidence.push({
|
|||
|
|
tool: "inspectionRunSingleCommand",
|
|||
|
|
summary: "无法执行命令（vault 中找不到设备凭据）",
|
|||
|
|
keyFindings: ["提示：请先在 Vault 中配置该设备的主机和凭据"],
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// --- Execute plan via independent SSH（独立建 SSH，不依赖已有终端）---
|
|||
|
|
const executedCmds: Array<{ command: string; output: string; ok: boolean; purpose?: string }> = [];
|
|||
|
|
for (const c of plan.commands) {
|
|||
|
|
if (!hostConfig) {
|
|||
|
|
executedCmds.push({
|
|||
|
|
command: c.command,
|
|||
|
|
output: "skipped — 无 hostConfig",
|
|||
|
|
ok: false,
|
|||
|
|
purpose: c.purpose,
|
|||
|
|
});
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
const start = performance.now();
|
|||
|
|
try {
|
|||
|
|
console.log("[RCA] exec cmd:", c.command, "on", hostConfig.hostname);
|
|||
|
|
// 优先走 preload inspection 命名空间，兜底直接走 ipcRenderer.invoke
|
|||
|
|
const b = bridge();
|
|||
|
|
let resp: any = null;
|
|||
|
|
if (b.inspection?.runSingleCommand) {
|
|||
|
|
resp = await b.inspection.runSingleCommand({
|
|||
|
|
hostConfig,
|
|||
|
|
command: c.command,
|
|||
|
|
timeoutMs: 15000,
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
// 兜底：直接调 ipcRenderer（preload 没注册时可能走这里）
|
|||
|
|
const w = window as any;
|
|||
|
|
if (w.netcatty?.ipcRenderer?.invoke) {
|
|||
|
|
resp = await w.netcatty.ipcRenderer.invoke("inspection:runSingleCommand", {
|
|||
|
|
hostConfig,
|
|||
|
|
command: c.command,
|
|||
|
|
timeoutMs: 15000,
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
throw new Error("inspection.runSingleCommand 不可用");
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
const rawOut = (resp?.stdout || "") + (resp?.stderr ? "\n--- stderr ---\n" + resp.stderr : "");
|
|||
|
|
const outStr = _stripVendorBanner(rawOut).slice(0, 5000);
|
|||
|
|
const rawFirst200 = rawOut.slice(0, 200).replace(/\n/g, "\\n");
|
|||
|
|
const cleanFirst200 = outStr.slice(0, 200).replace(/\n/g, "\\n");
|
|||
|
|
console.log("[RCA] cmd result:", resp);
|
|||
|
|
if (rawFirst200 !== cleanFirst200) {
|
|||
|
|
console.log("[RCA] vendor banner stripped:", { raw: rawFirst200, clean: cleanFirst200 });
|
|||
|
|
}
|
|||
|
|
executedCmds.push({
|
|||
|
|
command: c.command,
|
|||
|
|
output: outStr,
|
|||
|
|
ok: true,
|
|||
|
|
purpose: c.purpose,
|
|||
|
|
});
|
|||
|
|
invocations.push({
|
|||
|
|
tool: "inspectionRunCommand",
|
|||
|
|
ok: true,
|
|||
|
|
durationMs: Math.round(performance.now() - start),
|
|||
|
|
});
|
|||
|
|
} catch (e: any) {
|
|||
|
|
executedCmds.push({
|
|||
|
|
command: c.command,
|
|||
|
|
output: e?.message || String(e),
|
|||
|
|
ok: false,
|
|||
|
|
purpose: c.purpose,
|
|||
|
|
});
|
|||
|
|
invocations.push({
|
|||
|
|
tool: "inspectionRunCommand",
|
|||
|
|
ok: false,
|
|||
|
|
durationMs: Math.round(performance.now() - start),
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// --- Phase 2: AI 综合 RCA ---
|
|||
|
|
const phase2User = [
|
|||
|
|
`## 设备信息`,
|
|||
|
|
`- catalogId: ${catalogId || "unknown"} (${catalogLabel})`,
|
|||
|
|
`- 是否在线: ${online}`,
|
|||
|
|
``,
|
|||
|
|
`## 告警信息`,
|
|||
|
|
`- 设备: ${alert.device_name || alert.device_id}`,
|
|||
|
|
`- 规则: ${alert.rule_name} (${alert.rule_id})`,
|
|||
|
|
`- 指标: ${alert.metric} = ${alert.metric_value} (阈值 ${alert.threshold})`,
|
|||
|
|
`- 级别: ${alert.level}`,
|
|||
|
|
``,
|
|||
|
|
`## 最新巡检 metrics`,
|
|||
|
|
metrics?.metrics ? JSON.stringify(metrics.metrics, null, 2) : "(暂无数据)",
|
|||
|
|
``,
|
|||
|
|
`## 诊断命令执行结果`,
|
|||
|
|
...executedCmds.map(
|
|||
|
|
(c) =>
|
|||
|
|
`### ${c.command}${c.ok ? "" : " ❌ failed"}\n` +
|
|||
|
|
`用途: ${c.purpose || "-"}\n` +
|
|||
|
|
`\`\`\`\n${c.output.slice(0, 2500)}\n\`\`\``,
|
|||
|
|
),
|
|||
|
|
``,
|
|||
|
|
`请综合以上证据分析根因并输出 JSON。`,
|
|||
|
|
].join("\n");
|
|||
|
|
|
|||
|
|
const phase2Raw = await aiChatCompletion(provider, phase2System(), [{ role: "user", content: phase2User }]);
|
|||
|
|
if (!phase2Raw) return null;
|
|||
|
|
|
|||
|
|
const rca = parseFinalRCA(phase2Raw, alert, evidence);
|
|||
|
|
if (!rca) return null;
|
|||
|
|
|
|||
|
|
// Attach command outputs to evidence
|
|||
|
|
for (const c of executedCmds) {
|
|||
|
|
evidence.push({
|
|||
|
|
tool: c.command,
|
|||
|
|
summary: c.purpose || "诊断命令",
|
|||
|
|
keyFindings: [c.output.slice(0, 500)],
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
rca.evidence = evidence;
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
success: true,
|
|||
|
|
source: "ai",
|
|||
|
|
rca,
|
|||
|
|
rawText: phase2Raw,
|
|||
|
|
durationMs: 0,
|
|||
|
|
toolInvocations: invocations,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function parseFirstJSON(text: string): any {
|
|||
|
|
let s = text.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/i, "");
|
|||
|
|
try { return JSON.parse(s); } catch (_) {}
|
|||
|
|
const m = s.match(/\{[\s\S]*\}/);
|
|||
|
|
if (m) { try { return JSON.parse(m[0]); } catch (_) {} }
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function parseFinalRCA(text: string, alert: AlertRecord, evidence: RCAResult["evidence"]): RCAResult | null {
|
|||
|
|
const parsed = parseFirstJSON(text);
|
|||
|
|
if (!parsed || typeof parsed !== "object") return null;
|
|||
|
|
if (!parsed.rootCauses?.length && !parsed.summary) return null;
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
alertId: alert.id,
|
|||
|
|
deviceId: alert.device_id,
|
|||
|
|
deviceName: alert.device_name || alert.device_id,
|
|||
|
|
confidence: Math.max(0, Math.min(100, Number(parsed.confidence) || 0)),
|
|||
|
|
rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses.map(String) : [],
|
|||
|
|
evidence,
|
|||
|
|
recommendations: Array.isArray(parsed.recommendations)
|
|||
|
|
? parsed.recommendations.map((r: any) => ({
|
|||
|
|
priority: ["immediate", "short_term", "long_term"].includes(r?.priority) ? r.priority : "short_term",
|
|||
|
|
action: String(r?.action ?? ""),
|
|||
|
|
command: r?.command ? String(r.command) : undefined,
|
|||
|
|
risk: ["low", "medium", "high"].includes(r?.risk) ? r.risk : "medium",
|
|||
|
|
note: r?.note ? String(r.note) : undefined,
|
|||
|
|
}))
|
|||
|
|
: [],
|
|||
|
|
summary: String(parsed.summary ?? ""),
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- Deterministic ----------
|
|||
|
|
|
|||
|
|
async function runFallbackPath(
|
|||
|
|
alert: AlertRecord,
|
|||
|
|
metrics: any,
|
|||
|
|
invocations: RunRCAResult["toolInvocations"],
|
|||
|
|
): Promise<RCAResult> {
|
|||
|
|
const evidence: RCAResult["evidence"] = [];
|
|||
|
|
const commands = pickFallbackCmds(alert, metrics?.catalogId as string | undefined);
|
|||
|
|
|
|||
|
|
for (const cmd of commands) {
|
|||
|
|
const start = performance.now();
|
|||
|
|
try {
|
|||
|
|
const resp = (await rpc("netcatty/inspectionRunCommand", {
|
|||
|
|
deviceId: alert.device_id,
|
|||
|
|
command: cmd,
|
|||
|
|
timeoutMs: 15000,
|
|||
|
|
})) as any;
|
|||
|
|
const outStr = (resp?.stdout || "") + (resp?.stderr ? "\n--- stderr ---\n" + resp.stderr : "");
|
|||
|
|
evidence.push({ tool: cmd, summary: "诊断命令输出", keyFindings: [outStr.slice(0, 2000)] });
|
|||
|
|
invocations.push({ tool: "inspectionRunCommand", ok: true, durationMs: Math.round(performance.now() - start) });
|
|||
|
|
} catch (e: any) {
|
|||
|
|
evidence.push({ tool: cmd, summary: "命令执行失败", keyFindings: [e?.message || ""] });
|
|||
|
|
invocations.push({ tool: "inspectionRunCommand", ok: false, durationMs: Math.round(performance.now() - start) });
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const offline = alert.rule_id === "rule_offline";
|
|||
|
|
const cpu = alert.metric === "cpu_usage";
|
|||
|
|
const mem = alert.metric === "memory_usage";
|
|||
|
|
const disk = alert.metric === "disk_usage";
|
|||
|
|
const hostLabel = alert.device_name || alert.device_id;
|
|||
|
|
|
|||
|
|
const rootCauses: string[] = [];
|
|||
|
|
const recs: RCAResult["recommendations"] = [];
|
|||
|
|
let confidence = 40;
|
|||
|
|
|
|||
|
|
if (offline) {
|
|||
|
|
rootCauses.push("设备离线或 SSH 连接异常");
|
|||
|
|
confidence = 55;
|
|||
|
|
recs.push({ priority: "immediate", action: "检查网络连通性", command: `ping -c 3 ${hostLabel}`, risk: "low" });
|
|||
|
|
recs.push({ priority: "immediate", action: "确认 SSH 服务和端口", risk: "low" });
|
|||
|
|
}
|
|||
|
|
if (cpu) {
|
|||
|
|
rootCauses.push(`CPU ${alert.metric_value}% 超过阈值 ${alert.threshold}% — 热点进程或 I/O wait`);
|
|||
|
|
confidence = 60;
|
|||
|
|
recs.push({ priority: "immediate", action: "定位 CPU 热点进程", command: "top -bn1 | head -20 && ps aux --sort=-%cpu | head -10", risk: "low" });
|
|||
|
|
}
|
|||
|
|
if (mem) {
|
|||
|
|
rootCauses.push(`内存 ${alert.metric_value}% — 内存泄漏或突发负载`);
|
|||
|
|
confidence = 60;
|
|||
|
|
recs.push({ priority: "immediate", action: "检查 top 内存进程", command: "free -m && ps aux --sort=-%mem | head -10", risk: "low" });
|
|||
|
|
}
|
|||
|
|
if (disk) {
|
|||
|
|
rootCauses.push("磁盘使用率超限 — 日志堆积或大文件残留");
|
|||
|
|
confidence = 60;
|
|||
|
|
recs.push({ priority: "immediate", action: "定位大文件并清理", command: "df -h && du -sh /var/log/* /tmp/* | sort -rh | head -20", risk: "medium", note: "删除前确认不影响日志轮转" });
|
|||
|
|
}
|
|||
|
|
if (!rootCauses.length) {
|
|||
|
|
rootCauses.push("告警触发但信息不足，需要人工排查");
|
|||
|
|
confidence = 25;
|
|||
|
|
recs.push({ priority: "short_term", action: "人工登录设备检查", risk: "low" });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
alertId: alert.id,
|
|||
|
|
deviceId: alert.device_id,
|
|||
|
|
deviceName: hostLabel,
|
|||
|
|
confidence,
|
|||
|
|
rootCauses,
|
|||
|
|
evidence,
|
|||
|
|
recommendations: recs,
|
|||
|
|
summary: `${hostLabel} 触发 ${alert.level} 级告警（${alert.rule_name}）。根因: ${rootCauses[0]}。置信度 ${confidence}%（确定性分析，无 AI 参与）。`,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export function parseRCAResult(raw: string, alert: AlertRecord): RCAResult | null {
|
|||
|
|
return parseFinalRCA(raw, alert, []);
|
|||
|
|
}
|