Files
SenMeshNet/electron/bridges/inspection/alertRecordStore.cjs
zhaolei 1bb5390263
Some checks failed
AI automation / Route event (push) Has been cancelled
AI automation / Hand reopened issue to maintainers (push) Has been cancelled
AI automation / Clean source issue state (push) Has been cancelled
AI automation / Reconcile handoffs (push) Has been cancelled
AI automation / Classify issue (push) Has been cancelled
AI automation / Claude Code smoke (push) Has been cancelled
AI automation / Review issue follow-up (push) Has been cancelled
AI automation / Publish issue follow-up (push) Has been cancelled
AI automation / Implement with Claude Code (push) Has been cancelled
AI automation / Publish implement PR (push) Has been cancelled
AI automation / Continue queued issue comments (push) Has been cancelled
AI automation / Codex review loop (push) Has been cancelled
AI automation / Publish Codex fix (push) Has been cancelled
AI automation / Clear Codex dispatch marker (push) Has been cancelled
AI automation / Own PR re-request Codex (push) Has been cancelled
AI automation / External PR re-request Codex (push) Has been cancelled
AI automation / Poll Codex reaction / retry (push) Has been cancelled
docs: add system overview and plan-management-and-alert-remediation v1.3 design doc
- system-overview.md: full platform summary for NetMesh v2.x

- plan-management-and-alert-remediation.md: dual-track alert response design with 26 decisions

- alertRecordStore.cjs: remove auto_recover per D25 (alerts must be manually closed)
2026-10-30 16:35:50 +08:00

409 lines
14 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"use strict";
/**
* 告警记录存储
*
* 告警记录结构:
* {
* id: "alert_xxx",
* rule_id: "rule_cpu_critical",
* rule_name: "CPU 使用率过高",
* device_id: "host_xxx",
* device_name: "核心交换机-01",
* metric: "cpu_usage",
* metric_value: "85",
* threshold: "80",
* level: "critical",
* message: "核心交换机-01 CPU 使用率 85%,超过阈值 80%",
* triggered_at: timestamp,
* status: "active",
* acknowledged_by: null,
* acknowledged_at: null,
* resolved_at: null,
* resolved_reason: null,
* note: null,
* }
*/
const { STORAGE_KEY_ALERT_RECORDS, STORAGE_KEY_ALERT_COUNTERS } = require("./storageKeys.cjs");
const { ALERT_RECORD_STATUS, ALERT_LEVELS } = require("./types.cjs");
const MAX_ALERTS = 2000;
let _alerts = null;
let _counters = null; // 防抖动计数器 { "deviceId_ruleId": count }
function _ensureLoaded(readFn) {
if (_alerts === null) {
const raw = readFn(STORAGE_KEY_ALERT_RECORDS);
_alerts = raw ? _safeParse(raw, []) : [];
}
if (_counters === null) {
const raw = readFn(STORAGE_KEY_ALERT_COUNTERS);
_counters = raw ? _safeParse(raw, {}) : {};
}
}
function _safeParse(text, def) {
if (text === null || text === undefined) return def;
if (typeof text === "object") return text;
try { return JSON.parse(text); }
catch { return def; }
}
function _saveAlerts(writeFn) {
if (_alerts.length > MAX_ALERTS) {
// 只保留最近的
_alerts = _alerts.sort((a, b) => b.triggered_at - a.triggered_at).slice(0, MAX_ALERTS);
}
writeFn(STORAGE_KEY_ALERT_RECORDS, JSON.stringify(_alerts));
}
function _saveCounters(writeFn) {
writeFn(STORAGE_KEY_ALERT_COUNTERS, JSON.stringify(_counters));
}
// ---------- 告警触发(带防抖动) ----------
/**
* 处理告警:检查是否满足持续次数,满足则创建/更新告警记录
* @param {object} device - 设备信息 { device_id, device_name }
* @param {object} rule - 告警规则
* @param {number|string} metricValue - 当前指标值
* @param {boolean} isTriggered - 是否触发(指标值 > 阈值)
* @returns {object|null} 新创建或恢复的告警记录,无变化则返回 null
*/
function processAlert(readFn, writeFn, device, rule, metricValue, isTriggered) {
_ensureLoaded(readFn);
const counterKey = `${device.device_id}_${rule.id}`;
const activeAlert = _findActiveAlert(device.device_id, rule.id);
if (isTriggered) {
// 触发:计数 +1
_counters[counterKey] = (_counters[counterKey] || 0) + 1;
_saveCounters(writeFn);
// 达到持续次数且没有活跃告警 → 创建告警
if (_counters[counterKey] >= rule.duration && !activeAlert) {
const alert = _createAlert(device, rule, metricValue);
_alerts.push(alert);
_saveAlerts(writeFn);
return { ...alert, action: "triggered" };
}
// 已有活跃告警 → 更新指标值 + 刷新触发时间
if (activeAlert) {
// 如果告警之前被手动确认过(acknowledged)但设备继续超限,重置回活跃状态
if (activeAlert.status === ALERT_RECORD_STATUS.ACKNOWLEDGED) {
activeAlert.status = ALERT_RECORD_STATUS.ACTIVE;
activeAlert.acknowledged_by = null;
activeAlert.acknowledged_at = null;
}
activeAlert.metric_value = _stringifyMetric(metricValue);
activeAlert.threshold = _stringifyMetric(rule.threshold);
activeAlert.message = _buildMessage(device.device_name, rule, _stringifyMetric(metricValue), _stringifyMetric(rule.threshold));
activeAlert.triggered_at = Date.now();
_saveAlerts(writeFn);
return { ...activeAlert, action: "updated" };
}
return null;
} else {
// 未触发:重置计数
if (_counters[counterKey]) {
delete _counters[counterKey];
_saveCounters(writeFn);
}
// 注意:不再自动恢复告警(auto_recover 已移除)。
// 告警一旦触发就保持 ACTIVE / ACKNOWLEDGED,必须由值班员手动 resolveAlert() 或 closeAlert() 才会关闭。
// 原因:闪断/偶发正常会导致告警"闪了一下就没了",值班员还没来得及处理就消失了。
return null;
}
}
function _findActiveAlert(deviceId, ruleId) {
return _alerts.find(
(a) =>
a.device_id === deviceId &&
a.rule_id === ruleId &&
(a.status === ALERT_RECORD_STATUS.ACTIVE ||
a.status === ALERT_RECORD_STATUS.ACKNOWLEDGED)
);
}
function _stringifyMetric(v) {
if (v === null || v === undefined) return "";
if (typeof v === "number" || typeof v === "string" || typeof v === "boolean") return String(v);
if (Array.isArray(v)) {
// 数组:数字数组 join 逗号,对象数组 JSON 然后截断
if (v.length === 0) return "";
if (v.every(x => typeof x === "number" || typeof x === "string" || typeof x === "boolean")) {
return v.map(x => String(x)).join(", ");
}
try { return JSON.stringify(v).slice(0, 200); } catch { return ""; }
}
if (typeof v === "object") {
// 对象:优先取 value / current / usage 等常见数值字段
if (typeof v.value === "number" || typeof v.value === "string") return String(v.value);
if (typeof v.current === "number" || typeof v.current === "string") return String(v.current);
if (typeof v.usage === "number" || typeof v.usage === "string") return String(v.usage);
if (typeof v.percent === "number" || typeof v.percent === "string") return String(v.percent);
try { return JSON.stringify(v).slice(0, 200); } catch { return ""; }
}
return String(v);
}
function _createAlert(device, rule, metricValue) {
const mv = _stringifyMetric(metricValue);
const tv = _stringifyMetric(rule.threshold);
return {
id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
rule_id: rule.id,
rule_name: rule.name,
device_id: device.device_id,
device_name: device.device_name || device.device_id,
metric: rule.metric,
metric_value: mv,
threshold: tv,
level: rule.level,
message: _buildMessage(device.device_name || device.device_id, rule, mv, tv),
triggered_at: Date.now(),
status: ALERT_RECORD_STATUS.ACTIVE,
acknowledged_by: null,
acknowledged_at: null,
resolved_at: null,
resolved_reason: null,
note: null,
};
}
function _buildMessage(deviceName, rule, metricValueStr, thresholdStr) {
const unit = rule.metric.includes("_usage") || rule.metric.includes("_percent") ? "%" : "";
return `${deviceName} ${rule.name},当前值 ${metricValueStr}${unit},阈值 ${thresholdStr}${unit}`;
}
/**
* 创建执行错误告警(timeout / failed 等采集器执行异常)
* 每台设备 + rule_offline 全局唯一:
* - 已存在任何非 RESOLVED 状态 → 复用,强制激活,覆盖 message/triggered_at
* - 同设备多条残留(历史遗留)→ 自动合并:最新一条保留,其余降级为 RESOLVED
* - 全 RESOLVED → 新建一条 ACTIVE
*/
function createExecutionErrorAlert(readFn, writeFn, device, status, errorMessage) {
_ensureLoaded(readFn);
const offlineRuleId = "rule_offline";
const offlineRuleName = "设备离线";
// 1. 强制去重:同 device + rule_offline 的非 RESOLVED 告警,只保留最新的一条
const sameDeviceOffline = _alerts.filter(
(a) => a.device_id === device.device_id && a.rule_id === offlineRuleId
);
const activeList = sameDeviceOffline.filter(
(a) => a.status !== ALERT_RECORD_STATUS.RESOLVED && a.status !== ALERT_RECORD_STATUS.CLOSED
);
if (activeList.length > 1) {
// 多条残留 → triggered_at 最新的保留,其余强制降级为 RESOLVED(清理历史重复)
activeList.sort((a, b) => b.triggered_at - a.triggered_at);
for (let i = 1; i < activeList.length; i++) {
activeList[i].status = ALERT_RECORD_STATUS.RESOLVED;
activeList[i].resolved_at = Date.now();
activeList[i].resolved_reason = "dedup_execution_error";
}
_saveAlerts(writeFn);
}
// 2. 复用唯一剩余的那条非 RESOLVED 告警(ACTIVE 或 ACKNOWLEDGED 都算)
const existing = activeList[0];
if (existing && existing.status !== ALERT_RECORD_STATUS.RESOLVED && existing.status !== ALERT_RECORD_STATUS.CLOSED) {
if (existing.status === ALERT_RECORD_STATUS.ACKNOWLEDGED) {
existing.status = ALERT_RECORD_STATUS.ACTIVE;
existing.acknowledged_by = null;
existing.acknowledged_at = null;
}
existing.metric_value = "false";
existing.message = `${device.device_name || device.device_id} ${offlineRuleName}: ${errorMessage}`;
existing.triggered_at = Date.now();
_saveAlerts(writeFn);
return { ...existing, action: "updated" };
}
// 3. 没有 → 新建
const alert = {
id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
rule_id: offlineRuleId,
rule_name: offlineRuleName,
device_id: device.device_id,
device_name: device.device_name || device.device_id,
metric: "online",
metric_value: "false",
threshold: "true",
level: "critical",
message: `${device.device_name || device.device_id} ${offlineRuleName}: ${errorMessage}`,
triggered_at: Date.now(),
status: ALERT_RECORD_STATUS.ACTIVE,
acknowledged_by: null,
acknowledged_at: null,
resolved_at: null,
resolved_reason: null,
note: null,
};
_alerts.push(alert);
_saveAlerts(writeFn);
return { ...alert, action: "triggered" };
}
// ---------- 告警查询 ----------
function listAlerts(readFn, options = {}) {
_ensureLoaded(readFn);
let result = [..._alerts];
if (options.status) {
result = result.filter((a) => a.status === options.status);
}
// 读取时去重:同 device_id + rule_offline + (ACTIVE/ACKNOWLEDGED) 只留最新一条
// 这是对历史遗留重复数据的保护,正常流程已由 createExecutionErrorAlert 保证唯一
const offlineGroups = new Map();
result = result.filter((a) => {
if (a.rule_id === "rule_offline" && (a.status === "active" || a.status === "acknowledged")) {
const key = `${a.device_id}::offline-active`;
const prev = offlineGroups.get(key);
if (prev) {
// 已存在 → 保留触发时间新的那条
if (a.triggered_at > prev.triggered_at) {
offlineGroups.set(key, a);
return true; // 这条更新,过滤掉 prev(但 prev 已经在 result 里了,下面处理)
}
return false; // 这条旧,丢弃
}
offlineGroups.set(key, a);
}
return true;
});
// 二次清理:把每个 group 里最旧的那条也去掉
const bestOffline = new Set(offlineGroups.values().map((a) => a.id));
result = result.filter(
(a) => !(a.rule_id === "rule_offline" && (a.status === "active" || a.status === "acknowledged")) || bestOffline.has(a.id)
);
if (options.level) {
result = result.filter((a) => a.level === options.level);
}
if (options.device_id) {
result = result.filter((a) => a.device_id === options.device_id);
}
return result.sort((a, b) => b.triggered_at - a.triggered_at).slice(0, options.limit || MAX_ALERTS);
}
function getActiveAlerts(readFn) {
return listAlerts(readFn, { status: ALERT_RECORD_STATUS.ACTIVE, limit: MAX_ALERTS });
}
function getAlert(readFn, alertId) {
_ensureLoaded(readFn);
return _alerts.find((a) => a.id === alertId) || null;
}
// ---------- 告警处理 ----------
function acknowledgeAlert(readFn, writeFn, alertId, operator = "system") {
_ensureLoaded(readFn);
const alert = _alerts.find((a) => a.id === alertId);
if (!alert) return null;
alert.status = ALERT_RECORD_STATUS.ACKNOWLEDGED;
alert.acknowledged_by = operator;
alert.acknowledged_at = Date.now();
_saveAlerts(writeFn);
return { ...alert };
}
function resolveAlert(readFn, writeFn, alertId, reason = "manual", operator = "system") {
_ensureLoaded(readFn);
const alert = _alerts.find((a) => a.id === alertId);
if (!alert) return null;
alert.status = ALERT_RECORD_STATUS.RESOLVED;
alert.resolved_at = Date.now();
alert.resolved_reason = reason;
alert.acknowledged_by = alert.acknowledged_by || operator;
alert.acknowledged_at = alert.acknowledged_at || Date.now();
_saveAlerts(writeFn);
return { ...alert };
}
function closeAlert(readFn, writeFn, alertId, note = "") {
_ensureLoaded(readFn);
const alert = _alerts.find((a) => a.id === alertId);
if (!alert) return null;
alert.status = ALERT_RECORD_STATUS.CLOSED;
alert.note = note;
if (!alert.resolved_at) {
alert.resolved_at = Date.now();
alert.resolved_reason = "closed";
}
_saveAlerts(writeFn);
return { ...alert };
}
function updateNote(readFn, writeFn, alertId, note) {
_ensureLoaded(readFn);
const alert = _alerts.find((a) => a.id === alertId);
if (!alert) return null;
alert.note = note;
_saveAlerts(writeFn);
return { ...alert };
}
// ---------- 统计 ----------
function getAlertStats(readFn) {
_ensureLoaded(readFn);
const active = _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.ACTIVE);
return {
total: _alerts.length,
active: active.length,
active_critical: active.filter((a) => a.level === ALERT_LEVELS.CRITICAL).length,
active_warning: active.filter((a) => a.level === ALERT_LEVELS.WARNING).length,
active_info: active.filter((a) => a.level === ALERT_LEVELS.INFO).length,
acknowledged: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.ACKNOWLEDGED).length,
resolved: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.RESOLVED).length,
closed: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.CLOSED).length,
};
}
// ---------- 重置(测试用) ----------
function _resetForTest() {
_alerts = null;
_counters = null;
}
module.exports = {
// 告警触发
processAlert,
createExecutionErrorAlert,
// 查询
listAlerts,
getActiveAlerts,
getAlert,
// 处理
acknowledgeAlert,
resolveAlert,
closeAlert,
updateNote,
// 统计
getAlertStats,
// 内部
_resetForTest,
};