feat: 新增设备巡检模块实现完整的巡检告警功能
新增设备巡检模块核心功能，包括： - 巡检数据存储与调度系统 - 华为/H3C交换机及Linux服务器采集器 - 设备扩展配置与告警规则管理 - 主机详情页巡检配置入口 - 产品名称自定义设置功能 - 网络设备SSH exec通道兼容处理
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
/* eslint-disable no-undef */
|
||||
const { Client: SSHClient } = require("ssh2");
|
||||
const { buildAlgorithms } = require("../sshAlgorithms.cjs");
|
||||
const { executeBoundedSshCommand } = require("../boundedSshExec.cjs");
|
||||
const { listInteractiveShellPids } = require("../sshInteractiveShells.cjs");
|
||||
function decodeLsofFileName(value) {
|
||||
@@ -1006,6 +1008,350 @@ function createSessionOpsApi(ctx) {
|
||||
return { success: false, error: 'Session not found or not connected' };
|
||||
}
|
||||
|
||||
// ---------- NETWORK DEVICE STATS via independent SSH + PTY shell ----------
|
||||
// H3C / Huawei VRP / Cisco IOS / etc. only allow ONE SSH channel per
|
||||
// transport — an exec channel on the interactive conn drops the whole
|
||||
// connection. Open a SEPARATE SSH transport (independent Client, PTY
|
||||
// shell, paging disabled), run display commands, disconnect. This
|
||||
// mirrors the inspection executor pattern — no shared state with the
|
||||
// user's live session.
|
||||
const _v = session.remoteSshVersion || '';
|
||||
const isNetworkDeviceSsh = /^H3C[-_\s]/i.test(_v)
|
||||
|| /^Comware-/i.test(_v)
|
||||
|| /^HUAWEI[-_]/i.test(_v)
|
||||
|| /^VRP-/i.test(_v)
|
||||
|| _v === '-'
|
||||
|| /^Cisco[-_]|^CiscoIOS_/i.test(_v)
|
||||
|| /^RGOS_SSH/i.test(_v)
|
||||
|| /^JUNOS/i.test(_v)
|
||||
|| /H3C/i.test(_v) // 通用 H3C 检测
|
||||
|| /Comware/i.test(_v); // 通用 Comware 检测
|
||||
|
||||
if (isNetworkDeviceSsh && session._statsAuth?.hostname) {
|
||||
try {
|
||||
return await collectNetworkDeviceStats(session);
|
||||
} catch (err) {
|
||||
log("network device stats failed", { sessionId, error: err?.message || String(err) });
|
||||
return { success: false, error: err?.message || String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
async function collectNetworkDeviceStats(sess) {
|
||||
const auth = sess._statsAuth;
|
||||
const fs = require('fs');
|
||||
const logPath = require('path').join(require('os').tmpdir(), 'netmesh-stats-debug.log');
|
||||
const dlog = (m) => { try { fs.appendFileSync(logPath, `\n[${new Date().toISOString()}] ${m}`); } catch {} };
|
||||
dlog(`=== collectNetworkDeviceStats START hostname=${auth.hostname}`);
|
||||
|
||||
const connOpts = {
|
||||
host: auth.hostname,
|
||||
port: auth.port || 22,
|
||||
username: auth.username || 'root',
|
||||
readyTimeout: 15000,
|
||||
keepaliveInterval: 0,
|
||||
algorithms: buildAlgorithms(auth.legacyAlgorithms, {
|
||||
skipEcdsaHostKey: auth.skipEcdsaHostKey,
|
||||
}),
|
||||
};
|
||||
if (typeof auth.password === 'string' && auth.password.length > 0) {
|
||||
connOpts.password = auth.password;
|
||||
connOpts.tryKeyboard = true;
|
||||
}
|
||||
if (typeof auth.privateKey === 'string' && auth.privateKey.length > 0) {
|
||||
connOpts.privateKey = auth.privateKey;
|
||||
if (auth.passphrase) connOpts.passphrase = auth.passphrase;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const client = new SSHClient();
|
||||
let settled = false;
|
||||
let stream = null;
|
||||
let timer = null;
|
||||
|
||||
const finish = (result) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
dlog(`finish: ${JSON.stringify(result).slice(0, 200)}`);
|
||||
try { timer && clearTimeout(timer); } catch {}
|
||||
try { stream && stream.end(); } catch {}
|
||||
try { client.end(); } catch {}
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
client.once('banner', (msg) => dlog(`banner: ${msg.slice(0, 100)}`));
|
||||
client.once('handshake', () => dlog('handshake OK'));
|
||||
client.once('ready', () => {
|
||||
dlog('ready event → opening shell');
|
||||
client.shell({ term: 'vt100', cols: 120, rows: 40 }, (err, s) => {
|
||||
if (err) {
|
||||
dlog(`shell open FAILED: ${err.message}`);
|
||||
finish({ success: false, error: 'shell open failed: ' + err.message });
|
||||
return;
|
||||
}
|
||||
dlog('shell opened OK');
|
||||
stream = s;
|
||||
collectStatsOverShell(stream, finish, dlog);
|
||||
});
|
||||
});
|
||||
|
||||
client.once('error', (err) => {
|
||||
dlog(`error event: ${err?.message || String(err)} level=${err?.level}`);
|
||||
finish({ success: false, error: err?.message || String(err) });
|
||||
});
|
||||
client.once('close', () => {
|
||||
dlog(`close event settled=${settled}`);
|
||||
if (!settled) finish({ success: false, error: 'SSH connection closed' });
|
||||
});
|
||||
|
||||
try { client.connect(connOpts); dlog('connect() called'); }
|
||||
catch (err) { dlog(`connect() threw: ${err.message}`); finish({ success: false, error: err?.message || String(err) }); }
|
||||
});
|
||||
}
|
||||
|
||||
function collectStatsOverShell(stream, finish, dlog) {
|
||||
let phaseBuffer = '';
|
||||
let phase = 'init'; // init → disable-paging → cpu → mem → intf → quit → done
|
||||
let output = { cpu: null, memTotal: null, memUsed: null, memPercent: null, ports: [] };
|
||||
let phaseCompleted = false;
|
||||
|
||||
const PROMPT_RE = /^<[\w\-./]+>$|^[\w\-./]+[>#]$/;
|
||||
|
||||
const commands = [
|
||||
{ phase: 'disable-paging', cmd: 'screen-length 0 temporary', noParse: true },
|
||||
{ phase: 'cpu', cmd: 'display cpu-usage' },
|
||||
{ phase: 'mem', cmd: 'display memory' },
|
||||
{ phase: 'intf', cmd: 'display interface brief' },
|
||||
];
|
||||
let cmdIdx = 0;
|
||||
let globalTimeout = null;
|
||||
let initFired = false;
|
||||
let quitSent = false;
|
||||
|
||||
// --- Parsers ---
|
||||
function parseCpuUsage(text) {
|
||||
// H3C Comware 5/7: "7% in last 5 seconds" 或 "3% in last 1 minute"
|
||||
const m = text.match(/(\d+)\s*%\s*in\s+last\s+5\s+seconds/i)
|
||||
|| text.match(/(\d+)\s*%\s*in\s+last\s+1\s+minute/i)
|
||||
|| text.match(/5\s*Seconds\s*[:：]\s*(\d+)\s*%/i)
|
||||
|| text.match(/\(5-\d+\s*second\)\s*[:：]\s*(\d+)\s*%/i)
|
||||
|| text.match(/CPU\s+utilization[^.]*?(\d{1,3})\s*%/i);
|
||||
if (m) return parseInt(m[1], 10);
|
||||
return null;
|
||||
}
|
||||
function parseMemory(text) {
|
||||
// H3C Comware: "Used Rate: 44%" 或 "Memory Using Percentage Is: 21%"
|
||||
const pctM = text.match(/Used\s*Rate\s*[:：]?\s*(\d+(?:\.\d+)?)\s*%/i)
|
||||
|| text.match(/Memory\s+Using\s+Percentage\s+Is\s*[:：]?\s*(\d+(?:\.\d+)?)\s*%/i);
|
||||
const percent = pctM ? parseFloat(pctM[1]) : null;
|
||||
// bytes 格式: "System Total Memory(bytes): 65546160"
|
||||
const totalBytesM = text.match(/System\s+Total\s+Memory\s*\(\s*bytes\s*\)\s*[:：]?\s*(\d+)/i);
|
||||
const usedBytesM = text.match(/Total\s+Used\s+Memory\s*\(\s*bytes\s*\)\s*[:：]?\s*(\d+)/i);
|
||||
let total = null, used = null;
|
||||
if (totalBytesM && usedBytesM) {
|
||||
total = parseInt(totalBytesM[1], 10);
|
||||
used = parseInt(usedBytesM[1], 10);
|
||||
}
|
||||
// MB 格式: "Memory Size: xxx MB"
|
||||
const totalM = text.match(/Memory\s*(?:Size|Total)\s*[:：]?\s*(\d+)\s*MB/i);
|
||||
const usedM = text.match(/Memory\s*Used\s*[:：]?\s*(\d+)\s*MB/i);
|
||||
if (totalM && usedM) {
|
||||
total = parseInt(totalM[1], 10);
|
||||
used = parseInt(usedM[1], 10);
|
||||
}
|
||||
return {
|
||||
total,
|
||||
used,
|
||||
percent,
|
||||
};
|
||||
}
|
||||
function parseInterfaceBrief(text) {
|
||||
const ports = [];
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
// H3C 格式: "GE1/0/1 UP 100M(a)" 或 "GigabitEthernet1/0/1 UP UP"
|
||||
const geMatch = line.match(/^(GE\d+\/\d+\/\d+)\s+(UP|DOWN|up|down)/i);
|
||||
if (geMatch) {
|
||||
const phy = geMatch[2].toLowerCase();
|
||||
ports.push({ name: geMatch[1], phy: phy.includes('up') ? 'up' : 'down' });
|
||||
continue;
|
||||
}
|
||||
const m = line.match(/^([A-Za-z][\w\-]+(?:\d+[\/.])*\d+)\s+(UP|DOWN|up|down)/i);
|
||||
if (m && !/^(Name|Interface|Port)\b/i.test(m[1])) {
|
||||
const phy = (m[2] || '').toLowerCase();
|
||||
ports.push({ name: m[1], phy: phy.includes('up') ? 'up' : 'down' });
|
||||
}
|
||||
}
|
||||
return ports;
|
||||
}
|
||||
|
||||
function sendNextCommand() {
|
||||
clearTimeout(globalTimeout);
|
||||
if (cmdIdx >= commands.length) {
|
||||
dlog('ALL COMMANDS DONE → quit');
|
||||
// 标记 quit 阶段：此时交换机关连接是预期行为，不要当错误处理
|
||||
phase = 'quit';
|
||||
quitSent = true;
|
||||
try { stream.write('quit\n'); } catch {}
|
||||
// 交换机 quit 后会立刻关 VTY（close → error），等 1.5s 让它走完再 resolve
|
||||
globalTimeout = setTimeout(() => {
|
||||
dlog('quit timeout → resolve');
|
||||
resolveOutput();
|
||||
}, 1500);
|
||||
return;
|
||||
}
|
||||
const c = commands[cmdIdx];
|
||||
phase = c.phase;
|
||||
phaseBuffer = '';
|
||||
phaseCompleted = false;
|
||||
dlog(`>>> WRITE: ${c.cmd}`);
|
||||
try { stream.write(c.cmd + '\n'); } catch (err) { dlog(`write error: ${err.message}`); }
|
||||
// 每个 phase 10s timeout
|
||||
globalTimeout = setTimeout(() => {
|
||||
dlog(`phase ${phase} TIMEOUT, skip`);
|
||||
cmdIdx++;
|
||||
setTimeout(sendNextCommand, 200);
|
||||
}, 10000);
|
||||
}
|
||||
|
||||
function resolveOutput() {
|
||||
const upCount = output.ports.filter(p => p.phy === 'up').length;
|
||||
const downCount = output.ports.length - upCount;
|
||||
// 网络设备 display memory 通常只返回百分比（如 21%），没有 total/used MB
|
||||
// 构造假值让 ratioPercent(memUsed, memTotal) 能正确算出百分比
|
||||
let memTotal = output.memTotal;
|
||||
let memUsed = output.memUsed;
|
||||
if (memTotal == null && memUsed == null && output.memPercent != null) {
|
||||
memTotal = 100;
|
||||
memUsed = output.memPercent;
|
||||
}
|
||||
const full = {
|
||||
success: true,
|
||||
stats: {
|
||||
hostname: null, osName: null, kernelRelease: null, uptimeSeconds: null,
|
||||
loadAverage: [],
|
||||
cpu: output.cpu, cpuCores: null, cpuPerCore: [],
|
||||
memTotal, memUsed,
|
||||
memFree: memTotal != null && memUsed != null ? Math.max(0, memTotal - memUsed) : null,
|
||||
memBuffers: null, memCached: null, swapTotal: null, swapUsed: null,
|
||||
topProcesses: [], diskPercent: null, diskUsed: null, diskTotal: null, disks: [],
|
||||
netRxSpeed: 0, netTxSpeed: 0, latencyMs: null, netInterfaces: [],
|
||||
lastUpdated: Date.now(),
|
||||
networkDevicePorts: output.ports,
|
||||
networkDevicePortUp: upCount,
|
||||
networkDevicePortDown: downCount,
|
||||
},
|
||||
};
|
||||
dlog(`RESOLVE ✓ cpu=${output.cpu} memPercent=${output.memPercent} memTotal=${memTotal} memUsed=${memUsed} ports=${output.ports.length} up=${upCount} down=${downCount}`);
|
||||
dlog(`FULL JSON has networkDevicePorts? ${Array.isArray(full.stats.networkDevicePorts)} len=${full.stats.networkDevicePorts?.length}`);
|
||||
finish(full);
|
||||
}
|
||||
|
||||
stream.on('data', (data) => {
|
||||
const text = data.toString('utf-8');
|
||||
phaseBuffer += text;
|
||||
const lastLine = (phaseBuffer.split(/\r?\n/).pop() || '').trimEnd();
|
||||
|
||||
// 处理分页提示符
|
||||
if (phase === 'intf' && (phaseBuffer.includes('-- More --') || phaseBuffer.includes('---- More ----'))) {
|
||||
dlog('检测到分页，发送空格继续');
|
||||
stream.write(' ');
|
||||
// 移除分页提示符避免干扰
|
||||
phaseBuffer = phaseBuffer.replace(/--\s*More\s*--/gi, '').replace(/----\s*More\s*----/gi, '');
|
||||
}
|
||||
|
||||
if (!phaseCompleted && PROMPT_RE.test(lastLine)) {
|
||||
phaseCompleted = true;
|
||||
dlog(`PROMPT [${phase}]: ${lastLine}`);
|
||||
|
||||
if (phase === 'init') {
|
||||
// 第一次收到 shell prompt → 触发第一条命令
|
||||
// 关键：不要 cmdIdx++！直接 sendNextCommand 让它从 cmdIdx=0 开始
|
||||
initFired = true;
|
||||
setTimeout(sendNextCommand, 300);
|
||||
return;
|
||||
}
|
||||
|
||||
if (phase === 'cpu') {
|
||||
output.cpu = parseCpuUsage(phaseBuffer);
|
||||
dlog(` → cpu=${output.cpu}`);
|
||||
} else if (phase === 'mem') {
|
||||
const m = parseMemory(phaseBuffer);
|
||||
output.memTotal = m.total; output.memUsed = m.used; output.memPercent = m.percent;
|
||||
dlog(` → memTotal=${m.total} memUsed=${m.used} memPercent=${m.percent}`);
|
||||
} else if (phase === 'intf') {
|
||||
output.ports = parseInterfaceBrief(phaseBuffer);
|
||||
dlog(` → ports=${output.ports.length}`);
|
||||
}
|
||||
cmdIdx++;
|
||||
setTimeout(sendNextCommand, 200);
|
||||
}
|
||||
});
|
||||
|
||||
stream.on('close', () => {
|
||||
dlog(`stream close event quitSent=${quitSent}`);
|
||||
clearTimeout(globalTimeout);
|
||||
// quit 后交换机正常关 VTY → 直接 resolve，让后面的 client error 看到 settled=true
|
||||
if (quitSent) {
|
||||
dlog('→ quit-expected close, resolve now');
|
||||
resolveOutput();
|
||||
}
|
||||
});
|
||||
|
||||
// Init fallback: 5s 内没收到 shell prompt，也手动触发第一条
|
||||
setTimeout(() => {
|
||||
if (!initFired) {
|
||||
dlog('INIT FALLBACK fire (no shell prompt yet)');
|
||||
initFired = true;
|
||||
sendNextCommand();
|
||||
}
|
||||
}, 5000);
|
||||
|
||||
// 15s 全局超时（兜底）
|
||||
globalTimeout = setTimeout(() => finish({ success: false, error: 'network device stats timeout' }), 20000);
|
||||
}
|
||||
|
||||
function resolveNetworkDeviceStats(output, finish) {
|
||||
const upCount = output.ports.filter(p => p.phy === 'up').length;
|
||||
const downCount = output.ports.length - upCount;
|
||||
// Map to the standard ServerStats shape the frontend expects.
|
||||
// Fields we cannot derive from H3C/Huawei display commands (processes,
|
||||
// disks, per-core CPU, load average, sparkline samples) are left null.
|
||||
finish({
|
||||
success: true,
|
||||
stats: {
|
||||
hostname: null,
|
||||
osName: null,
|
||||
kernelRelease: null,
|
||||
uptimeSeconds: null,
|
||||
loadAverage: [],
|
||||
cpu: output.cpu,
|
||||
cpuCores: null,
|
||||
cpuPerCore: [],
|
||||
memTotal: output.memTotal,
|
||||
memUsed: output.memUsed,
|
||||
memFree: output.memTotal && output.memUsed ? Math.max(0, output.memTotal - output.memUsed) : null,
|
||||
memBuffers: null,
|
||||
memCached: null,
|
||||
swapTotal: null,
|
||||
swapUsed: null,
|
||||
topProcesses: [],
|
||||
diskPercent: null,
|
||||
diskUsed: null,
|
||||
diskTotal: null,
|
||||
disks: [],
|
||||
netRxSpeed: 0,
|
||||
netTxSpeed: 0,
|
||||
latencyMs: null,
|
||||
netInterfaces: [],
|
||||
lastUpdated: Date.now(),
|
||||
// Extended fields for network devices (the POSIX path never sets
|
||||
// these — they are purely for panel rendering):
|
||||
networkDevicePorts: output.ports,
|
||||
networkDevicePortUp: upCount,
|
||||
networkDevicePortDown: downCount,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const isEtSession = session.type === "et";
|
||||
const etUsesExecFallback = isEtSession && session.etStatsAuth?.hasJumpHost;
|
||||
|
||||
|
||||
@@ -345,6 +345,31 @@ function createStartSessionApi(ctx) {
|
||||
: false,
|
||||
cols: options.cols || 80,
|
||||
rows: options.rows || 24,
|
||||
// Reusable auth bag for background companion connections (network
|
||||
// device stats PTY shell, etc.). Only non-sensitive fields that the
|
||||
// moshStatsConnection helper also needs — do NOT store the raw options
|
||||
// object here (it contains callbacks, DOM refs, and other non-serializable
|
||||
// junk). This is deliberately best-effort: any auth field missing here
|
||||
// just means the companion skips and stats stay empty — the interactive
|
||||
// shell keeps working regardless.
|
||||
_statsAuth: {
|
||||
hostname: options.host || options.hostname || '',
|
||||
port: options.port || 22,
|
||||
username: options.username || '',
|
||||
password: typeof options.password === 'string' ? options.password : undefined,
|
||||
privateKey: typeof options.privateKey === 'string' ? options.privateKey : undefined,
|
||||
passphrase: typeof options.passphrase === 'string' ? options.passphrase : undefined,
|
||||
authMethod: options.authMethod || undefined,
|
||||
knownHosts: Array.isArray(options.knownHosts) ? options.knownHosts : undefined,
|
||||
verifyHostKeys: options.verifyHostKeys !== false,
|
||||
legacyAlgorithms: !!options.legacyAlgorithms,
|
||||
skipEcdsaHostKey: !!options.skipEcdsaHostKey,
|
||||
algorithmOverrides: options.algorithmOverrides,
|
||||
useSshAgent: options.useSshAgent,
|
||||
identityAgent: options.identityAgent,
|
||||
certificate: typeof options.certificate === 'string' ? options.certificate : undefined,
|
||||
keyId: typeof options.keyId === 'string' ? options.keyId : undefined,
|
||||
},
|
||||
};
|
||||
const { claimSessionSlot } = require("../sessionBootEpoch.cjs");
|
||||
const claim = claimSessionSlot(sessions, sessionId, session, options.bootEpoch);
|
||||
|
||||
Reference in New Issue
Block a user