Initial commit: NetMesh smart ops platform with 3D topology, inspection reports, RCA and AI ops assistant
Some checks failed
AI automation / Route event (push) Has been cancelled
AI automation / Hand reopened issue to maintainers (push) Has been cancelled
AI automation / Clean source issue state (push) Has been cancelled
AI automation / Reconcile handoffs (push) Has been cancelled
AI automation / Classify issue (push) Has been cancelled
AI automation / Claude Code smoke (push) Has been cancelled
AI automation / Review issue follow-up (push) Has been cancelled
AI automation / Publish issue follow-up (push) Has been cancelled
AI automation / Implement with Claude Code (push) Has been cancelled
AI automation / Publish implement PR (push) Has been cancelled
AI automation / Continue queued issue comments (push) Has been cancelled
AI automation / Codex review loop (push) Has been cancelled
AI automation / Publish Codex fix (push) Has been cancelled
AI automation / Clear Codex dispatch marker (push) Has been cancelled
AI automation / Own PR re-request Codex (push) Has been cancelled
AI automation / External PR re-request Codex (push) Has been cancelled
AI automation / Poll Codex reaction / retry (push) Has been cancelled
Some checks failed
AI automation / Route event (push) Has been cancelled
AI automation / Hand reopened issue to maintainers (push) Has been cancelled
AI automation / Clean source issue state (push) Has been cancelled
AI automation / Reconcile handoffs (push) Has been cancelled
AI automation / Classify issue (push) Has been cancelled
AI automation / Claude Code smoke (push) Has been cancelled
AI automation / Review issue follow-up (push) Has been cancelled
AI automation / Publish issue follow-up (push) Has been cancelled
AI automation / Implement with Claude Code (push) Has been cancelled
AI automation / Publish implement PR (push) Has been cancelled
AI automation / Continue queued issue comments (push) Has been cancelled
AI automation / Codex review loop (push) Has been cancelled
AI automation / Publish Codex fix (push) Has been cancelled
AI automation / Clear Codex dispatch marker (push) Has been cancelled
AI automation / Own PR re-request Codex (push) Has been cancelled
AI automation / External PR re-request Codex (push) Has been cancelled
AI automation / Poll Codex reaction / retry (push) Has been cancelled
This commit is contained in:
411
electron/bridges/inspection/alertRecordStore.cjs
Normal file
411
electron/bridges/inspection/alertRecordStore.cjs
Normal file
@@ -0,0 +1,411 @@
|
||||
"use strict";
|
||||
|
||||
/**
|
||||
* 告警记录存储
|
||||
*
|
||||
* 告警记录结构:
|
||||
* {
|
||||
* id: "alert_xxx",
|
||||
* rule_id: "rule_cpu_critical",
|
||||
* rule_name: "CPU 使用率过高",
|
||||
* device_id: "host_xxx",
|
||||
* device_name: "核心交换机-01",
|
||||
* metric: "cpu_usage",
|
||||
* metric_value: "85",
|
||||
* threshold: "80",
|
||||
* level: "critical",
|
||||
* message: "核心交换机-01 CPU 使用率 85%,超过阈值 80%",
|
||||
* triggered_at: timestamp,
|
||||
* status: "active",
|
||||
* acknowledged_by: null,
|
||||
* acknowledged_at: null,
|
||||
* resolved_at: null,
|
||||
* resolved_reason: null,
|
||||
* note: null,
|
||||
* }
|
||||
*/
|
||||
|
||||
const { STORAGE_KEY_ALERT_RECORDS, STORAGE_KEY_ALERT_COUNTERS } = require("./storageKeys.cjs");
|
||||
const { ALERT_RECORD_STATUS, ALERT_LEVELS } = require("./types.cjs");
|
||||
|
||||
const MAX_ALERTS = 2000;
|
||||
|
||||
let _alerts = null;
|
||||
let _counters = null; // 防抖动计数器 { "deviceId_ruleId": count }
|
||||
|
||||
function _ensureLoaded(readFn) {
|
||||
if (_alerts === null) {
|
||||
const raw = readFn(STORAGE_KEY_ALERT_RECORDS);
|
||||
_alerts = raw ? _safeParse(raw, []) : [];
|
||||
}
|
||||
if (_counters === null) {
|
||||
const raw = readFn(STORAGE_KEY_ALERT_COUNTERS);
|
||||
_counters = raw ? _safeParse(raw, {}) : {};
|
||||
}
|
||||
}
|
||||
|
||||
function _safeParse(text, def) {
|
||||
try { return JSON.parse(text); }
|
||||
catch (e) { return def; }
|
||||
}
|
||||
|
||||
function _saveAlerts(writeFn) {
|
||||
if (_alerts.length > MAX_ALERTS) {
|
||||
// 只保留最近的
|
||||
_alerts = _alerts.sort((a, b) => b.triggered_at - a.triggered_at).slice(0, MAX_ALERTS);
|
||||
}
|
||||
writeFn(STORAGE_KEY_ALERT_RECORDS, JSON.stringify(_alerts));
|
||||
}
|
||||
|
||||
function _saveCounters(writeFn) {
|
||||
writeFn(STORAGE_KEY_ALERT_COUNTERS, JSON.stringify(_counters));
|
||||
}
|
||||
|
||||
// ---------- 告警触发(带防抖动) ----------
|
||||
|
||||
/**
|
||||
* 处理告警:检查是否满足持续次数,满足则创建/更新告警记录
|
||||
* @param {object} device - 设备信息 { device_id, device_name }
|
||||
* @param {object} rule - 告警规则
|
||||
* @param {number|string} metricValue - 当前指标值
|
||||
* @param {boolean} isTriggered - 是否触发(指标值 > 阈值)
|
||||
* @returns {object|null} 新创建或恢复的告警记录,无变化则返回 null
|
||||
*/
|
||||
function processAlert(readFn, writeFn, device, rule, metricValue, isTriggered) {
|
||||
_ensureLoaded(readFn);
|
||||
|
||||
const counterKey = `${device.device_id}_${rule.id}`;
|
||||
const activeAlert = _findActiveAlert(device.device_id, rule.id);
|
||||
|
||||
if (isTriggered) {
|
||||
// 触发:计数 +1
|
||||
_counters[counterKey] = (_counters[counterKey] || 0) + 1;
|
||||
_saveCounters(writeFn);
|
||||
|
||||
// 达到持续次数且没有活跃告警 → 创建告警
|
||||
if (_counters[counterKey] >= rule.duration && !activeAlert) {
|
||||
const alert = _createAlert(device, rule, metricValue);
|
||||
_alerts.push(alert);
|
||||
_saveAlerts(writeFn);
|
||||
return { ...alert, action: "triggered" };
|
||||
}
|
||||
|
||||
// 已有活跃告警 → 更新指标值 + 刷新触发时间
|
||||
if (activeAlert) {
|
||||
// 如果告警之前被手动确认过(acknowledged)但设备继续超限,重置回活跃状态
|
||||
if (activeAlert.status === ALERT_RECORD_STATUS.ACKNOWLEDGED) {
|
||||
activeAlert.status = ALERT_RECORD_STATUS.ACTIVE;
|
||||
activeAlert.acknowledged_by = null;
|
||||
activeAlert.acknowledged_at = null;
|
||||
}
|
||||
activeAlert.metric_value = _stringifyMetric(metricValue);
|
||||
activeAlert.threshold = _stringifyMetric(rule.threshold);
|
||||
activeAlert.message = _buildMessage(device.device_name, rule, _stringifyMetric(metricValue), _stringifyMetric(rule.threshold));
|
||||
activeAlert.triggered_at = Date.now();
|
||||
_saveAlerts(writeFn);
|
||||
return { ...activeAlert, action: "updated" };
|
||||
}
|
||||
|
||||
return null;
|
||||
} else {
|
||||
// 未触发:重置计数
|
||||
if (_counters[counterKey]) {
|
||||
delete _counters[counterKey];
|
||||
_saveCounters(writeFn);
|
||||
}
|
||||
|
||||
// 有活跃告警 → 自动恢复(ACTIVE 和 ACKNOWLEDGED 都要恢复)
|
||||
if (activeAlert && (activeAlert.status === ALERT_RECORD_STATUS.ACTIVE || activeAlert.status === ALERT_RECORD_STATUS.ACKNOWLEDGED)) {
|
||||
activeAlert.status = ALERT_RECORD_STATUS.RESOLVED;
|
||||
activeAlert.resolved_at = Date.now();
|
||||
activeAlert.resolved_reason = "auto_recover";
|
||||
_saveAlerts(writeFn);
|
||||
return { ...activeAlert, action: "resolved" };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function _findActiveAlert(deviceId, ruleId) {
|
||||
return _alerts.find(
|
||||
(a) =>
|
||||
a.device_id === deviceId &&
|
||||
a.rule_id === ruleId &&
|
||||
(a.status === ALERT_RECORD_STATUS.ACTIVE ||
|
||||
a.status === ALERT_RECORD_STATUS.ACKNOWLEDGED)
|
||||
);
|
||||
}
|
||||
|
||||
function _stringifyMetric(v) {
|
||||
if (v === null || v === undefined) return "";
|
||||
if (typeof v === "number" || typeof v === "string" || typeof v === "boolean") return String(v);
|
||||
if (Array.isArray(v)) {
|
||||
// 数组:数字数组 join 逗号,对象数组 JSON 然后截断
|
||||
if (v.length === 0) return "";
|
||||
if (v.every(x => typeof x === "number" || typeof x === "string" || typeof x === "boolean")) {
|
||||
return v.map(x => String(x)).join(", ");
|
||||
}
|
||||
try { return JSON.stringify(v).slice(0, 200); } catch { return ""; }
|
||||
}
|
||||
if (typeof v === "object") {
|
||||
// 对象:优先取 value / current / usage 等常见数值字段
|
||||
if (typeof v.value === "number" || typeof v.value === "string") return String(v.value);
|
||||
if (typeof v.current === "number" || typeof v.current === "string") return String(v.current);
|
||||
if (typeof v.usage === "number" || typeof v.usage === "string") return String(v.usage);
|
||||
if (typeof v.percent === "number" || typeof v.percent === "string") return String(v.percent);
|
||||
try { return JSON.stringify(v).slice(0, 200); } catch { return ""; }
|
||||
}
|
||||
return String(v);
|
||||
}
|
||||
|
||||
function _createAlert(device, rule, metricValue) {
|
||||
const mv = _stringifyMetric(metricValue);
|
||||
const tv = _stringifyMetric(rule.threshold);
|
||||
return {
|
||||
id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
||||
rule_id: rule.id,
|
||||
rule_name: rule.name,
|
||||
device_id: device.device_id,
|
||||
device_name: device.device_name || device.device_id,
|
||||
metric: rule.metric,
|
||||
metric_value: mv,
|
||||
threshold: tv,
|
||||
level: rule.level,
|
||||
message: _buildMessage(device.device_name || device.device_id, rule, mv, tv),
|
||||
triggered_at: Date.now(),
|
||||
status: ALERT_RECORD_STATUS.ACTIVE,
|
||||
acknowledged_by: null,
|
||||
acknowledged_at: null,
|
||||
resolved_at: null,
|
||||
resolved_reason: null,
|
||||
note: null,
|
||||
};
|
||||
}
|
||||
|
||||
function _buildMessage(deviceName, rule, metricValueStr, thresholdStr) {
|
||||
const unit = rule.metric.includes("_usage") || rule.metric.includes("_percent") ? "%" : "";
|
||||
return `${deviceName} ${rule.name},当前值 ${metricValueStr}${unit},阈值 ${thresholdStr}${unit}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建执行错误告警(timeout / failed 等采集器执行异常)
|
||||
* 每台设备 + rule_offline 全局唯一:
|
||||
* - 已存在任何非 RESOLVED 状态 → 复用,强制激活,覆盖 message/triggered_at
|
||||
* - 同设备多条残留(历史遗留)→ 自动合并:最新一条保留,其余降级为 RESOLVED
|
||||
* - 全 RESOLVED → 新建一条 ACTIVE
|
||||
*/
|
||||
function createExecutionErrorAlert(readFn, writeFn, device, status, errorMessage) {
|
||||
_ensureLoaded(readFn);
|
||||
|
||||
const offlineRuleId = "rule_offline";
|
||||
const offlineRuleName = "设备离线";
|
||||
|
||||
// 1. 强制去重:同 device + rule_offline 的非 RESOLVED 告警,只保留最新的一条
|
||||
const sameDeviceOffline = _alerts.filter(
|
||||
(a) => a.device_id === device.device_id && a.rule_id === offlineRuleId
|
||||
);
|
||||
const activeList = sameDeviceOffline.filter(
|
||||
(a) => a.status !== ALERT_RECORD_STATUS.RESOLVED && a.status !== ALERT_RECORD_STATUS.CLOSED
|
||||
);
|
||||
if (activeList.length > 1) {
|
||||
// 多条残留 → triggered_at 最新的保留,其余强制降级为 RESOLVED(清理历史重复)
|
||||
activeList.sort((a, b) => b.triggered_at - a.triggered_at);
|
||||
for (let i = 1; i < activeList.length; i++) {
|
||||
activeList[i].status = ALERT_RECORD_STATUS.RESOLVED;
|
||||
activeList[i].resolved_at = Date.now();
|
||||
activeList[i].resolved_reason = "dedup_execution_error";
|
||||
}
|
||||
_saveAlerts(writeFn);
|
||||
}
|
||||
|
||||
// 2. 复用唯一剩余的那条非 RESOLVED 告警(ACTIVE 或 ACKNOWLEDGED 都算)
|
||||
const existing = activeList[0];
|
||||
if (existing && existing.status !== ALERT_RECORD_STATUS.RESOLVED && existing.status !== ALERT_RECORD_STATUS.CLOSED) {
|
||||
if (existing.status === ALERT_RECORD_STATUS.ACKNOWLEDGED) {
|
||||
existing.status = ALERT_RECORD_STATUS.ACTIVE;
|
||||
existing.acknowledged_by = null;
|
||||
existing.acknowledged_at = null;
|
||||
}
|
||||
existing.metric_value = "false";
|
||||
existing.message = `${device.device_name || device.device_id} ${offlineRuleName}: ${errorMessage}`;
|
||||
existing.triggered_at = Date.now();
|
||||
_saveAlerts(writeFn);
|
||||
return { ...existing, action: "updated" };
|
||||
}
|
||||
|
||||
// 3. 没有 → 新建
|
||||
const alert = {
|
||||
id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
||||
rule_id: offlineRuleId,
|
||||
rule_name: offlineRuleName,
|
||||
device_id: device.device_id,
|
||||
device_name: device.device_name || device.device_id,
|
||||
metric: "online",
|
||||
metric_value: "false",
|
||||
threshold: "true",
|
||||
level: "critical",
|
||||
message: `${device.device_name || device.device_id} ${offlineRuleName}: ${errorMessage}`,
|
||||
triggered_at: Date.now(),
|
||||
status: ALERT_RECORD_STATUS.ACTIVE,
|
||||
acknowledged_by: null,
|
||||
acknowledged_at: null,
|
||||
resolved_at: null,
|
||||
resolved_reason: null,
|
||||
note: null,
|
||||
};
|
||||
|
||||
_alerts.push(alert);
|
||||
_saveAlerts(writeFn);
|
||||
return { ...alert, action: "triggered" };
|
||||
}
|
||||
|
||||
// ---------- 告警查询 ----------
|
||||
|
||||
function listAlerts(readFn, options = {}) {
|
||||
_ensureLoaded(readFn);
|
||||
let result = [..._alerts];
|
||||
|
||||
if (options.status) {
|
||||
result = result.filter((a) => a.status === options.status);
|
||||
}
|
||||
|
||||
// 读取时去重:同 device_id + rule_offline + (ACTIVE/ACKNOWLEDGED) 只留最新一条
|
||||
// 这是对历史遗留重复数据的保护,正常流程已由 createExecutionErrorAlert 保证唯一
|
||||
const offlineGroups = new Map();
|
||||
result = result.filter((a) => {
|
||||
if (a.rule_id === "rule_offline" && (a.status === "active" || a.status === "acknowledged")) {
|
||||
const key = `${a.device_id}::offline-active`;
|
||||
const prev = offlineGroups.get(key);
|
||||
if (prev) {
|
||||
// 已存在 → 保留触发时间新的那条
|
||||
if (a.triggered_at > prev.triggered_at) {
|
||||
offlineGroups.set(key, a);
|
||||
return true; // 这条更新,过滤掉 prev(但 prev 已经在 result 里了,下面处理)
|
||||
}
|
||||
return false; // 这条旧,丢弃
|
||||
}
|
||||
offlineGroups.set(key, a);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
// 二次清理:把每个 group 里最旧的那条也去掉
|
||||
const bestOffline = new Set(offlineGroups.values().map((a) => a.id));
|
||||
result = result.filter(
|
||||
(a) => !(a.rule_id === "rule_offline" && (a.status === "active" || a.status === "acknowledged")) || bestOffline.has(a.id)
|
||||
);
|
||||
|
||||
if (options.level) {
|
||||
result = result.filter((a) => a.level === options.level);
|
||||
}
|
||||
if (options.device_id) {
|
||||
result = result.filter((a) => a.device_id === options.device_id);
|
||||
}
|
||||
|
||||
return result.sort((a, b) => b.triggered_at - a.triggered_at).slice(0, options.limit || MAX_ALERTS);
|
||||
}
|
||||
|
||||
function getActiveAlerts(readFn) {
|
||||
return listAlerts(readFn, { status: ALERT_RECORD_STATUS.ACTIVE, limit: MAX_ALERTS });
|
||||
}
|
||||
|
||||
function getAlert(readFn, alertId) {
|
||||
_ensureLoaded(readFn);
|
||||
return _alerts.find((a) => a.id === alertId) || null;
|
||||
}
|
||||
|
||||
// ---------- 告警处理 ----------
|
||||
|
||||
function acknowledgeAlert(readFn, writeFn, alertId, operator = "system") {
|
||||
_ensureLoaded(readFn);
|
||||
const alert = _alerts.find((a) => a.id === alertId);
|
||||
if (!alert) return null;
|
||||
|
||||
alert.status = ALERT_RECORD_STATUS.ACKNOWLEDGED;
|
||||
alert.acknowledged_by = operator;
|
||||
alert.acknowledged_at = Date.now();
|
||||
_saveAlerts(writeFn);
|
||||
return { ...alert };
|
||||
}
|
||||
|
||||
function resolveAlert(readFn, writeFn, alertId, reason = "manual", operator = "system") {
|
||||
_ensureLoaded(readFn);
|
||||
const alert = _alerts.find((a) => a.id === alertId);
|
||||
if (!alert) return null;
|
||||
|
||||
alert.status = ALERT_RECORD_STATUS.RESOLVED;
|
||||
alert.resolved_at = Date.now();
|
||||
alert.resolved_reason = reason;
|
||||
alert.acknowledged_by = alert.acknowledged_by || operator;
|
||||
alert.acknowledged_at = alert.acknowledged_at || Date.now();
|
||||
_saveAlerts(writeFn);
|
||||
return { ...alert };
|
||||
}
|
||||
|
||||
function closeAlert(readFn, writeFn, alertId, note = "") {
|
||||
_ensureLoaded(readFn);
|
||||
const alert = _alerts.find((a) => a.id === alertId);
|
||||
if (!alert) return null;
|
||||
|
||||
alert.status = ALERT_RECORD_STATUS.CLOSED;
|
||||
alert.note = note;
|
||||
if (!alert.resolved_at) {
|
||||
alert.resolved_at = Date.now();
|
||||
alert.resolved_reason = "closed";
|
||||
}
|
||||
_saveAlerts(writeFn);
|
||||
return { ...alert };
|
||||
}
|
||||
|
||||
function updateNote(readFn, writeFn, alertId, note) {
|
||||
_ensureLoaded(readFn);
|
||||
const alert = _alerts.find((a) => a.id === alertId);
|
||||
if (!alert) return null;
|
||||
|
||||
alert.note = note;
|
||||
_saveAlerts(writeFn);
|
||||
return { ...alert };
|
||||
}
|
||||
|
||||
// ---------- 统计 ----------
|
||||
|
||||
function getAlertStats(readFn) {
|
||||
_ensureLoaded(readFn);
|
||||
|
||||
const active = _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.ACTIVE);
|
||||
return {
|
||||
total: _alerts.length,
|
||||
active: active.length,
|
||||
active_critical: active.filter((a) => a.level === ALERT_LEVELS.CRITICAL).length,
|
||||
active_warning: active.filter((a) => a.level === ALERT_LEVELS.WARNING).length,
|
||||
active_info: active.filter((a) => a.level === ALERT_LEVELS.INFO).length,
|
||||
acknowledged: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.ACKNOWLEDGED).length,
|
||||
resolved: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.RESOLVED).length,
|
||||
closed: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.CLOSED).length,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- 重置(测试用) ----------
|
||||
function _resetForTest() {
|
||||
_alerts = null;
|
||||
_counters = null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// 告警触发
|
||||
processAlert,
|
||||
createExecutionErrorAlert,
|
||||
// 查询
|
||||
listAlerts,
|
||||
getActiveAlerts,
|
||||
getAlert,
|
||||
// 处理
|
||||
acknowledgeAlert,
|
||||
resolveAlert,
|
||||
closeAlert,
|
||||
updateNote,
|
||||
// 统计
|
||||
getAlertStats,
|
||||
// 内部
|
||||
_resetForTest,
|
||||
};
|
||||
Reference in New Issue
Block a user