feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交
核心功能: - 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制 - FastAPI + PostgreSQL + Redis + Docker Compose 架构 - 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性 - 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页) - 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min) 并发与稳定性修复: - 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行) - JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写 - 移除 SQLite fallback, 统一 PostgreSQL - API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
11
app/diagnostic/__init__.py
Normal file
11
app/diagnostic/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""MeshSafe Diagnostics Kit v1.0 — 运行时模块。"""
|
||||
|
||||
from app.diagnostic.config import DiagConfig, get_diag_config
|
||||
from app.diagnostic.engine import DiagnosticEngine, RuleResult
|
||||
|
||||
__all__ = [
|
||||
"DiagConfig",
|
||||
"get_diag_config",
|
||||
"DiagnosticEngine",
|
||||
"RuleResult",
|
||||
]
|
||||
171
app/diagnostic/config.py
Normal file
171
app/diagnostic/config.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""MeshSafe Diagnostics Kit — 配置读取。
|
||||
|
||||
所有配置项可被环境变量覆盖(``MESHSAFE_DIAG_*`` 前缀),
|
||||
优先级:环境变量 > diagnostics.yaml > 默认值。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
_DEFAULT_YAML = _PROJECT_ROOT / "config" / "diagnostics.yaml"
|
||||
|
||||
|
||||
def _load_yaml(path: Path) -> Dict[str, Any]:
|
||||
"""读取 YAML 配置,文件不存在返回空 dict。"""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return data.get("diagnostics", {}) if isinstance(data, dict) else {}
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
v = os.environ.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
return v.strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
v = os.environ.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
try:
|
||||
return int(v)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _env_str(name: str, default: str) -> str:
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_diag_config() -> "DiagConfig":
|
||||
yaml_data = _load_yaml(_DEFAULT_YAML)
|
||||
return DiagConfig.from_dict(yaml_data)
|
||||
|
||||
|
||||
class DiagConfig:
|
||||
"""诊断套件配置。"""
|
||||
|
||||
def __init__(self, yaml_data: Dict[str, Any]):
|
||||
# 顶层开关
|
||||
yaml_enabled = yaml_data.get("enabled", True)
|
||||
self.enabled = _env_bool(
|
||||
"MESHSAFE_DIAG_ENABLED", yaml_enabled
|
||||
)
|
||||
|
||||
# Docker 子能力
|
||||
docker_yaml = yaml_data.get("docker", {})
|
||||
yaml_docker_mode = docker_yaml.get("enabled", "auto")
|
||||
self.docker_mode = _env_str(
|
||||
"MESHSAFE_DIAG_DOCKER", yaml_docker_mode
|
||||
).lower()
|
||||
if self.docker_mode not in ("auto", "true", "false"):
|
||||
self.docker_mode = "auto"
|
||||
|
||||
# 定时器周期(秒)
|
||||
timers_yaml = yaml_data.get("timers", {})
|
||||
self.error_scan_interval = _env_int(
|
||||
"MESHSAFE_DIAG_ERROR_INTERVAL",
|
||||
timers_yaml.get("error_scan_interval", 30),
|
||||
)
|
||||
self.container_check_interval = _env_int(
|
||||
"MESHSAFE_DIAG_CONTAINER_INTERVAL",
|
||||
timers_yaml.get("container_check_interval", 300),
|
||||
)
|
||||
self.db_aggregate_interval = _env_int(
|
||||
"MESHSAFE_DIAG_DB_INTERVAL",
|
||||
timers_yaml.get("db_aggregate_interval", 60),
|
||||
)
|
||||
|
||||
# 日志行数
|
||||
self.docker_log_tail = int(yaml_data.get("docker_log_tail", 500))
|
||||
self.export_log_tail = int(yaml_data.get("export_log_tail", 2000))
|
||||
|
||||
# 导出包数据范围
|
||||
export_yaml = yaml_data.get("export", {})
|
||||
self.export_db_span_days = int(export_yaml.get("db_span_days", 7))
|
||||
self.export_scan_tasks_max = int(export_yaml.get("scan_tasks_max", 10000))
|
||||
self.export_scan_findings_max = int(export_yaml.get("scan_findings_max", 50000))
|
||||
self.export_engine_errors_max = int(export_yaml.get("engine_errors_max", 5000))
|
||||
|
||||
# 规则阈值
|
||||
rules_yaml = yaml_data.get("rules", {})
|
||||
self.error_ratio_threshold = float(
|
||||
rules_yaml.get("error_ratio_threshold", 0.05)
|
||||
)
|
||||
self.clamav_db_stale_days = int(
|
||||
rules_yaml.get("clamav_db_stale_days", 7)
|
||||
)
|
||||
self.reject_ratio_threshold = float(
|
||||
rules_yaml.get("reject_ratio_threshold", 0.20)
|
||||
)
|
||||
self.suspicious_exceeds_malicious = bool(
|
||||
rules_yaml.get("suspicious_exceeds_malicious", True)
|
||||
)
|
||||
|
||||
# 持久化存储
|
||||
storage_yaml = yaml_data.get("storage", {})
|
||||
self.storage_enabled = _env_bool(
|
||||
"MESHSAFE_DIAG_STORAGE_ENABLED",
|
||||
storage_yaml.get("enabled", True),
|
||||
)
|
||||
self.storage_dir = storage_yaml.get("dir", "/app/data/diagnostics")
|
||||
self.history_retention_days = int(
|
||||
storage_yaml.get("history_retention_days", 7)
|
||||
)
|
||||
self.log_snapshot_on_timer = bool(
|
||||
storage_yaml.get("log_snapshot_on_timer", True)
|
||||
)
|
||||
self.log_snapshot_tail = int(
|
||||
storage_yaml.get("log_snapshot_tail", 500)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "DiagConfig":
|
||||
return cls(data)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"enabled": self.enabled,
|
||||
"docker_mode": self.docker_mode,
|
||||
"timers": {
|
||||
"error_scan_interval": self.error_scan_interval,
|
||||
"container_check_interval": self.container_check_interval,
|
||||
"db_aggregate_interval": self.db_aggregate_interval,
|
||||
},
|
||||
"docker_log_tail": self.docker_log_tail,
|
||||
"export_log_tail": self.export_log_tail,
|
||||
"export": {
|
||||
"db_span_days": self.export_db_span_days,
|
||||
"scan_tasks_max": self.export_scan_tasks_max,
|
||||
"scan_findings_max": self.export_scan_findings_max,
|
||||
"engine_errors_max": self.export_engine_errors_max,
|
||||
},
|
||||
"rules": {
|
||||
"error_ratio_threshold": self.error_ratio_threshold,
|
||||
"clamav_db_stale_days": self.clamav_db_stale_days,
|
||||
"reject_ratio_threshold": self.reject_ratio_threshold,
|
||||
"suspicious_exceeds_malicious": self.suspicious_exceeds_malicious,
|
||||
},
|
||||
"storage": {
|
||||
"enabled": self.storage_enabled,
|
||||
"dir": self.storage_dir,
|
||||
"history_retention_days": self.history_retention_days,
|
||||
"log_snapshot_on_timer": self.log_snapshot_on_timer,
|
||||
"log_snapshot_tail": self.log_snapshot_tail,
|
||||
},
|
||||
}
|
||||
126
app/diagnostic/docker_probe.py
Normal file
126
app/diagnostic/docker_probe.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""启动时 Docker 能力检测(用 python docker SDK)。
|
||||
|
||||
返回三档:
|
||||
full — docker.sock 可访问,日志和 inspect 都能用
|
||||
partial — 能连 socket 但权限受限
|
||||
none — 完全不可用(没挂 socket / 包没装)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
_PROBE_CONTAINER = "meshsafe-api"
|
||||
|
||||
|
||||
def _try_client() -> Optional["docker.DockerClient"]:
|
||||
"""尝试创建 Docker client。失败返回 None。"""
|
||||
try:
|
||||
import docker # noqa: F401
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
try:
|
||||
import docker
|
||||
# from_env() 会自动读 DOCKER_HOST,没设的话走 unix:///var/run/docker.sock
|
||||
client = docker.from_env(timeout=5)
|
||||
client.ping() # 真正连一下
|
||||
return client
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def detect_docker_access(mode: str = "auto") -> Dict[str, Any]:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
sock_exists = os.path.exists("/var/run/docker.sock")
|
||||
client = _try_client()
|
||||
|
||||
if mode == "false":
|
||||
return {
|
||||
"mode": "false", "status": "none",
|
||||
"docker.sock_exists": sock_exists,
|
||||
"docker_python_available": client is not None,
|
||||
"can_logs": False, "can_inspect": False,
|
||||
"containers": None,
|
||||
"message": "配置强制关闭 Docker 能力(diagnostics.docker.enabled=false)",
|
||||
"detected_at": now,
|
||||
}
|
||||
|
||||
if client is None:
|
||||
if mode == "true":
|
||||
msg = "配置强制启用但 Docker Python SDK 不可用 — pip install docker"
|
||||
else:
|
||||
msg = "Docker SDK 未就绪 — 日志匹配/容器状态类规则已跳过"
|
||||
return {
|
||||
"mode": mode, "status": "none",
|
||||
"docker.sock_exists": sock_exists,
|
||||
"docker_python_available": False,
|
||||
"can_logs": False, "can_inspect": False,
|
||||
"containers": None,
|
||||
"message": msg,
|
||||
"detected_at": now,
|
||||
}
|
||||
|
||||
# 真测
|
||||
can_logs = False
|
||||
can_inspect = False
|
||||
containers: Optional[List[str]] = None
|
||||
|
||||
try:
|
||||
c = client.containers.get(_PROBE_CONTAINER)
|
||||
c.logs(tail=1)
|
||||
can_logs = True
|
||||
c.attrs # 读取属性
|
||||
can_inspect = True
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
try:
|
||||
containers = [c.name for c in client.containers.list(all=True)]
|
||||
except Exception: # noqa: BLE001
|
||||
containers = None
|
||||
|
||||
if can_logs and can_inspect:
|
||||
status = "full"
|
||||
msg = "Docker 访问正常 — 日志匹配 + 容器状态检测全部启用"
|
||||
elif can_logs or can_inspect:
|
||||
status = "partial"
|
||||
msg = f"Docker SDK 可用但权限受限(logs={can_logs}, inspect={can_inspect})"
|
||||
else:
|
||||
status = "none"
|
||||
msg = "docker.sock 未挂或权限不足 — 日志匹配类规则已跳过"
|
||||
|
||||
if mode == "true" and status != "full":
|
||||
msg = f"配置强制启用但检测结果为 {status} — 检查 docker.sock 挂载"
|
||||
|
||||
return {
|
||||
"mode": mode, "status": status,
|
||||
"docker.sock_exists": sock_exists,
|
||||
"docker_python_available": True,
|
||||
"can_logs": can_logs,
|
||||
"can_inspect": can_inspect,
|
||||
"containers": containers,
|
||||
"message": msg,
|
||||
"detected_at": now,
|
||||
}
|
||||
|
||||
|
||||
def status_message(status: str) -> str:
|
||||
return {
|
||||
"full": "✅ Docker 能力完整",
|
||||
"partial": "⚠️ Docker 访问受限",
|
||||
"none": "ℹ️ Docker 日志检测未启用",
|
||||
}.get(status, status)
|
||||
|
||||
|
||||
def get_client():
|
||||
"""获取共享 docker client(给 dumper / rules 用)。失败返回 None。"""
|
||||
try:
|
||||
import docker
|
||||
return docker.from_env(timeout=10)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
477
app/diagnostic/dumper.py
Normal file
477
app/diagnostic/dumper.py
Normal file
@@ -0,0 +1,477 @@
|
||||
"""一键导出诊断包(diagnostics-YYYYMMDD-HHMMSS.tar.gz)。
|
||||
|
||||
调用方式(HTTP 接口):
|
||||
GET /api/v1/telemetry/dump → 返回文件下载
|
||||
|
||||
打包内容见方案文档 §3.4。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.diagnostic.config import get_diag_config
|
||||
from app.diagnostic.engine import DiagnosticEngine
|
||||
from app.diagnostic.docker_probe import detect_docker_access
|
||||
|
||||
|
||||
def _sanitize_filename(name: str) -> str:
|
||||
"""文件名脱敏:保留扩展名 + SHA256 前 8 位。"""
|
||||
if not name:
|
||||
return ""
|
||||
ext = os.path.splitext(name)[1].lower()
|
||||
# 截取前 20 字符 + hash-like 后缀
|
||||
return f"file_{hash(name) % 999999:06d}{ext}"
|
||||
|
||||
|
||||
def _sanitize_ip(ip: str) -> str:
|
||||
if not ip:
|
||||
return ""
|
||||
parts = ip.split(".")
|
||||
if len(parts) == 4:
|
||||
return f"{'.'.join(parts[:3])}.*"
|
||||
return ip
|
||||
|
||||
|
||||
def _bj_time(dt) -> Optional[str]:
|
||||
"""把 DB 里的 UTC datetime(naive 或 aware)转北京时间字符串。"""
|
||||
if dt is None:
|
||||
return None
|
||||
bj = timezone(timedelta(hours=8))
|
||||
try:
|
||||
# naive UTC → 加上 tz 再转
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
bj_dt = dt.astimezone(bj)
|
||||
return bj_dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception:
|
||||
return str(dt)
|
||||
|
||||
|
||||
def _collect_scan_tasks(days: int = 7, max_rows: int = 10000) -> List[Dict[str, Any]]:
|
||||
"""拉过去 N 天的 ScanTask,脱敏后返回 dict 列表。有 max_rows 兜底。"""
|
||||
from app.models import ScanTask
|
||||
db = SessionLocal()
|
||||
try:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
rows = (
|
||||
db.query(ScanTask)
|
||||
.filter(ScanTask.created_at >= cutoff)
|
||||
.order_by(ScanTask.created_at.desc())
|
||||
.limit(max_rows)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"filename_sanitized": _sanitize_filename(r.filename),
|
||||
"declared_ext": r.declared_ext,
|
||||
"size": r.size,
|
||||
"sha256": r.sha256[:16],
|
||||
"channel": r.channel,
|
||||
"status": r.status,
|
||||
"mode": r.mode,
|
||||
"verdict": r.verdict,
|
||||
"action": r.action,
|
||||
"score": r.score,
|
||||
"reason": r.reason[:500] if r.reason else "",
|
||||
"policy_version": r.policy_version,
|
||||
"created_at": _bj_time(r.created_at),
|
||||
"finished_at": _bj_time(r.finished_at),
|
||||
"duration_ms": r.duration_ms,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _collect_scan_findings(days: int = 7, max_rows: int = 50000) -> List[Dict[str, Any]]:
|
||||
from app.models import ScanFinding
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# ScanFinding 没有 created_at 字段?查 ScanTask 关联时间
|
||||
from app.models import ScanTask
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
rows = (
|
||||
db.query(ScanFinding)
|
||||
.join(ScanTask, ScanFinding.task_id == ScanTask.id)
|
||||
.filter(ScanTask.created_at >= cutoff)
|
||||
.order_by(ScanFinding.id.desc())
|
||||
.limit(max_rows)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"task_id": r.task_id,
|
||||
"engine": r.engine,
|
||||
"layer": r.layer,
|
||||
"severity": r.severity,
|
||||
"rule": r.rule,
|
||||
"weight": r.weight,
|
||||
"detail": (r.detail[:2000] if r.detail else ""),
|
||||
"extracted_path": r.extracted_path,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _collect_engine_errors(days: int = 7, max_rows: int = 5000) -> List[Dict[str, Any]]:
|
||||
from app.models import ScanTask
|
||||
db = SessionLocal()
|
||||
try:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
rows = (
|
||||
db.query(ScanTask)
|
||||
.filter(ScanTask.verdict == "error")
|
||||
.filter(ScanTask.created_at >= cutoff)
|
||||
.order_by(ScanTask.created_at.desc())
|
||||
.limit(max_rows)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"filename_sanitized": _sanitize_filename(r.filename),
|
||||
"reason": r.reason[:500] if r.reason else "",
|
||||
"created_at": _bj_time(r.created_at),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _collect_policy_snapshot() -> Dict[str, Any]:
|
||||
"""策略版本 + YARA 规则列表(不含规则原文)。"""
|
||||
from app.main import policy_store
|
||||
policy = policy_store.current()
|
||||
settings = get_settings()
|
||||
rules_dir = settings.rules_dir
|
||||
|
||||
yara_rules: List[str] = []
|
||||
if os.path.isdir(rules_dir):
|
||||
yara_rules = sorted(
|
||||
f for f in os.listdir(rules_dir)
|
||||
if f.endswith((".yar", ".yara", ".rules"))
|
||||
)
|
||||
|
||||
return {
|
||||
"version": policy.version,
|
||||
"sha256": policy.sha256,
|
||||
"yara_rules_dir": rules_dir,
|
||||
"yara_rules_count": len(yara_rules),
|
||||
"yara_rules_list": yara_rules[:50], # 最多 50 个文件名
|
||||
"global": policy.raw.get("global", {}),
|
||||
}
|
||||
|
||||
|
||||
def _collect_container_logs(containers: List[str], tail: int) -> Dict[str, str]:
|
||||
"""用 docker SDK 取日志。"""
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
client = get_client()
|
||||
result: Dict[str, str] = {}
|
||||
for c in containers:
|
||||
try:
|
||||
container = client.containers.get(c)
|
||||
raw = container.logs(tail=tail)
|
||||
result[c] = raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
|
||||
except Exception as e: # noqa: BLE001
|
||||
result[c] = f"[导出失败: {e}]"
|
||||
return result
|
||||
|
||||
|
||||
def _collect_container_meta(containers: List[str]) -> Dict[str, Dict[str, Any]]:
|
||||
"""用 docker SDK 取容器状态元数据(含 uptime / OOM)。"""
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
client = get_client()
|
||||
meta: Dict[str, Dict[str, Any]] = {}
|
||||
now = datetime.now(timezone.utc)
|
||||
for c in containers:
|
||||
try:
|
||||
container = client.containers.get(c)
|
||||
attrs = container.attrs or {}
|
||||
state = attrs.get("State", {})
|
||||
started_raw = state.get("StartedAt", "")
|
||||
started_at = _docker_time(started_raw)
|
||||
uptime_s = (now - started_at).total_seconds() if started_at else -1
|
||||
|
||||
meta[c] = {
|
||||
"status": state.get("Status", "unknown"),
|
||||
"health": state.get("Health", {}).get("Status", ""),
|
||||
"restart_count": attrs.get("RestartCount", -1),
|
||||
"oom_killed": state.get("OOMKilled", False),
|
||||
"exit_code": state.get("ExitCode", 0),
|
||||
"started_at": _bj_time(started_at),
|
||||
"uptime": _fmt_duration(uptime_s),
|
||||
"uptime_seconds": int(uptime_s) if uptime_s >= 0 else -1,
|
||||
"image": attrs.get("Config", {}).get("Image", ""),
|
||||
}
|
||||
except Exception as e: # noqa: BLE001
|
||||
meta[c] = {"error": str(e)[:200]}
|
||||
return meta
|
||||
|
||||
|
||||
def _collect_compose_state() -> Dict[str, Any]:
|
||||
"""用 docker SDK 列容器(替代 docker compose ps,输出结构化 JSON)。"""
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
client = get_client()
|
||||
now = datetime.now(timezone.utc)
|
||||
items: List[Dict[str, Any]] = []
|
||||
try:
|
||||
for c in client.containers.list(all=True):
|
||||
attrs = c.attrs or {}
|
||||
state = attrs.get("State", {})
|
||||
started_raw = state.get("StartedAt", "")
|
||||
started_at = _docker_time(started_raw)
|
||||
uptime_s = (now - started_at).total_seconds() if started_at else -1
|
||||
|
||||
items.append({
|
||||
"name": c.name,
|
||||
"status": state.get("Status", "unknown"),
|
||||
"health": state.get("Health", {}).get("Status", ""),
|
||||
"restart_count": attrs.get("RestartCount", -1),
|
||||
"oom_killed": state.get("OOMKilled", False),
|
||||
"exit_code": state.get("ExitCode", 0),
|
||||
"image": c.image.tags[0] if c.image.tags else "unknown",
|
||||
"started_at": _bj_time(started_at),
|
||||
"uptime": _fmt_duration(uptime_s),
|
||||
})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {"error": str(e)[:300], "containers": []}
|
||||
return {"containers": items, "count": len(items)}
|
||||
|
||||
|
||||
def _docker_time(raw: str) -> Optional[datetime]:
|
||||
"""把 docker 返回的 ISO 时间串转 aware UTC datetime。"""
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
# docker 返回: "2026-10-28T05:00:12.123456789Z" 或 ".123Z"
|
||||
s = raw.replace("Z", "+00:00")
|
||||
# 截断纳秒到微秒(Python fromisoformat 不接受纳秒)
|
||||
if "." in s:
|
||||
head, frac = s.split(".", 1)
|
||||
tz_idx = frac.rfind("+")
|
||||
if tz_idx >= 0:
|
||||
frac_ns = frac[:tz_idx]
|
||||
tz = frac[tz_idx:]
|
||||
frac_ns = (frac_ns + "000000")[:6]
|
||||
s = head + "." + frac_ns + tz
|
||||
return datetime.fromisoformat(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _fmt_duration(seconds: float) -> str:
|
||||
"""把秒数格式化成人类可读的时长。"""
|
||||
if seconds < 0:
|
||||
return "未知"
|
||||
s = int(seconds)
|
||||
days, rem = divmod(s, 86400)
|
||||
hours, rem = divmod(rem, 3600)
|
||||
mins, secs = divmod(rem, 60)
|
||||
if days > 0:
|
||||
return f"{days}d {hours}h {mins}m"
|
||||
if hours > 0:
|
||||
return f"{hours}h {mins}m"
|
||||
if mins > 0:
|
||||
return f"{mins}m {secs}s"
|
||||
return f"{secs}s"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 主入口:build_diagnostics_package
|
||||
# ======================================================================
|
||||
|
||||
def build_diagnostics_package() -> bytes:
|
||||
"""生成完整诊断包,返回 tar.gz 字节。"""
|
||||
|
||||
started = time.time()
|
||||
now = datetime.now(timezone.utc)
|
||||
cfg = get_diag_config()
|
||||
settings = get_settings()
|
||||
|
||||
containers = [
|
||||
"meshsafe-api", "meshsafe-postgres",
|
||||
"meshsafe-redis", "meshsafe-clamav", "meshsafe-nginx",
|
||||
]
|
||||
|
||||
# ---- 1. docker 能力 ----
|
||||
docker_info = detect_docker_access(mode=cfg.docker_mode)
|
||||
|
||||
# ---- 2. 导出前先存当前 docker 日志快照(持久化目录里)----
|
||||
snapshot_files: List[Path] = []
|
||||
try:
|
||||
from app.diagnostic.storage import snapshot_container_logs
|
||||
if cfg.storage_enabled and docker_info["status"] != "none":
|
||||
snapshot_files = snapshot_container_logs(
|
||||
containers, cfg.export_log_tail,
|
||||
filename_prefix="export",
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
snapshot_files = []
|
||||
|
||||
# ---- 3. 规则引擎(全量跑一遍)----
|
||||
db = SessionLocal()
|
||||
from app.main import scanner, policy_store
|
||||
eng = DiagnosticEngine(
|
||||
db=db, scanner=scanner, policy_store=policy_store,
|
||||
docker_status=docker_info["status"],
|
||||
docker_log_tail=cfg.docker_log_tail,
|
||||
)
|
||||
rule_results = eng.run_all(run_by="export")
|
||||
db.close()
|
||||
|
||||
# ---- 4. 加载规则历史(持久化 JSONL)----
|
||||
rule_history: List[Dict[str, Any]] = []
|
||||
restart_events: List[Dict[str, Any]] = []
|
||||
try:
|
||||
from app.diagnostic.storage import load_rule_history, load_restart_events
|
||||
if cfg.storage_enabled:
|
||||
rule_history = load_rule_history(cfg.history_retention_days)
|
||||
restart_events = load_restart_events(cfg.history_retention_days)
|
||||
except Exception: # noqa: BLE001
|
||||
rule_history = []
|
||||
restart_events = []
|
||||
|
||||
# 按容器聚合重启事件数(用于 rules_report 的 summary)
|
||||
restart_by_container: Dict[str, int] = {}
|
||||
for ev in restart_events:
|
||||
c = ev.get("container", "?")
|
||||
restart_by_container[c] = restart_by_container.get(c, 0) + 1
|
||||
|
||||
# ---- 5. DB 数据(默认过去 7 天,各有上限兜底)----
|
||||
span_days = getattr(cfg, "export_db_span_days", 7)
|
||||
scan_tasks = _collect_scan_tasks(
|
||||
days=span_days, max_rows=getattr(cfg, "export_scan_tasks_max", 10000),
|
||||
)
|
||||
scan_findings = _collect_scan_findings(
|
||||
days=span_days, max_rows=getattr(cfg, "export_scan_findings_max", 50000),
|
||||
)
|
||||
engine_errors = _collect_engine_errors(
|
||||
days=span_days, max_rows=getattr(cfg, "export_engine_errors_max", 5000),
|
||||
)
|
||||
policy_snapshot = _collect_policy_snapshot()
|
||||
|
||||
# ---- 6. 容器数据(如果 Docker 可用)----
|
||||
docker_logs_out: Dict[str, str] = {}
|
||||
container_meta: Dict[str, Dict[str, Any]] = {}
|
||||
compose_state: Dict[str, Any] = {"containers": [], "error": "Docker 不可用"}
|
||||
if docker_info["status"] != "none":
|
||||
docker_logs_out = _collect_container_logs(containers, cfg.export_log_tail)
|
||||
container_meta = _collect_container_meta(containers)
|
||||
compose_state = _collect_compose_state()
|
||||
|
||||
# ---- 7. 组装 tar.gz ----
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
||||
|
||||
def _add_json(name: str, obj: Any) -> None:
|
||||
data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8")
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
|
||||
def _add_text(name: str, text: str) -> None:
|
||||
data = text.encode("utf-8")
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
|
||||
# meta.json
|
||||
_add_json("meta.json", {
|
||||
"exported_at": now.isoformat(),
|
||||
"generated_by": "MeshSafe Diagnostics Kit v1.0",
|
||||
"meshsafe_version": "1.1.0",
|
||||
"docker_image": container_meta.get("meshsafe-api", {}).get("image", "unknown"),
|
||||
"policy_version": policy_snapshot["version"],
|
||||
"storage_enabled": cfg.storage_enabled,
|
||||
"rules_history_count": len(rule_history),
|
||||
"log_snapshots_included": len(snapshot_files),
|
||||
"export_duration_ms": int((time.time() - started) * 1000),
|
||||
# 数据范围
|
||||
"db_span_days": getattr(cfg, "export_db_span_days", 7),
|
||||
"scan_tasks_count": len(scan_tasks),
|
||||
"scan_findings_count": len(scan_findings),
|
||||
"engine_errors_count": len(engine_errors),
|
||||
})
|
||||
|
||||
# docker_detect.json
|
||||
_add_json("docker_detect.json", docker_info)
|
||||
|
||||
# compose_state.json(结构化 JSON)
|
||||
_add_json("compose_state.json", compose_state)
|
||||
|
||||
# rules_report.json(实时全量 + 持久化历史 + 重启事件汇总)
|
||||
_add_json("rules_report.json", {
|
||||
"exported_at": now.isoformat(),
|
||||
"docker_status": docker_info["status"],
|
||||
"current_run": [r.to_dict() for r in rule_results],
|
||||
"history_retention_days": cfg.history_retention_days,
|
||||
"history_count": len(rule_history),
|
||||
"history": rule_history,
|
||||
"restart_events_count": len(restart_events),
|
||||
"restart_events_summary": restart_by_container,
|
||||
})
|
||||
|
||||
# restart_events.json(完整重启事件历史)
|
||||
_add_json("restart_events.json", {
|
||||
"exported_at": now.isoformat(),
|
||||
"retention_days": cfg.history_retention_days,
|
||||
"count": len(restart_events),
|
||||
"events": restart_events,
|
||||
})
|
||||
|
||||
# scan_tasks.json / scan_findings.json / engine_errors.json
|
||||
_add_json("scan_tasks.json", scan_tasks)
|
||||
_add_json("scan_findings.json", scan_findings)
|
||||
_add_json("engine_errors.json", engine_errors)
|
||||
|
||||
# policy_snapshot.json
|
||||
_add_json("policy_snapshot.json", policy_snapshot)
|
||||
|
||||
# engine_health.json
|
||||
_add_json("engine_health.json", scanner.health())
|
||||
|
||||
# containers/*.log / *.meta.json(当前快照)
|
||||
for c in containers:
|
||||
if c in docker_logs_out:
|
||||
_add_text(f"containers/{c}.log", docker_logs_out[c])
|
||||
if c in container_meta:
|
||||
_add_json(f"containers/{c}.meta.json", container_meta[c])
|
||||
|
||||
# diagnostics_logs/(持久化目录下所有 .log — timer 快照 + export 快照 + restart 快照)
|
||||
try:
|
||||
from app.diagnostic.storage import _logs_dir
|
||||
logs_dir = _logs_dir()
|
||||
if cfg.storage_enabled and logs_dir.exists():
|
||||
for fpath in logs_dir.glob("*.log"):
|
||||
try:
|
||||
tar.add(
|
||||
str(fpath),
|
||||
arcname=f"diagnostics_logs/{fpath.name}",
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
except Exception: # noqa: BLE001
|
||||
# 回退:只用 export 生成的快照
|
||||
for fpath in snapshot_files:
|
||||
try:
|
||||
tar.add(str(fpath), arcname=f"diagnostics_logs/{fpath.name}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return buf.getvalue()
|
||||
228
app/diagnostic/engine.py
Normal file
228
app/diagnostic/engine.py
Normal file
@@ -0,0 +1,228 @@
|
||||
"""诊断规则引擎。
|
||||
|
||||
职责:
|
||||
- 定义 RuleResult 数据结构(每条规则的输出格式)
|
||||
- 提供 DiagnosticEngine.run_all() 入口
|
||||
- 规则实现分散在 rules.py 中,engine 负责调度
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuleResult:
|
||||
"""单条规则的输出。"""
|
||||
|
||||
id: str # R001, R002 ...
|
||||
severity: str # high / medium / low / info / ok
|
||||
message: str # 给用户看的中文描述
|
||||
hint: str = "" # 建议/下一步操作
|
||||
evidence: List[str] = field(default_factory=list) # 证据片段(日志行/DB 统计)
|
||||
run_at: str = "" # ISO 格式时间戳
|
||||
run_by: str = "page_request" # page_request / timer:xxx / export
|
||||
skipped: bool = False # 规则是否跳过
|
||||
skip_reason: Optional[str] = None # 为什么跳过(如 Docker 不可用)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.run_at:
|
||||
# 用北京时间(Asia/Shanghai, UTC+8),避免显示差 8 小时
|
||||
try:
|
||||
bj = timezone(timedelta(hours=8))
|
||||
self.run_at = datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception: # noqa: BLE001
|
||||
self.run_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 严重性 → 徽章 CSS 类映射(和 _nav_style.html 保持一致)
|
||||
# ---------------------------------------------------------------------------
|
||||
SEVERITY_BADGE: Dict[str, str] = {
|
||||
"high": "tag bad", # 红色
|
||||
"medium": "tag warn", # 橙色
|
||||
"low": "tag notice", # 黄色
|
||||
"info": "tag muted", # 灰色
|
||||
"ok": "tag ok", # 绿色
|
||||
}
|
||||
|
||||
|
||||
def badge_for(severity: str) -> str:
|
||||
return SEVERITY_BADGE.get(severity, "tag plain")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DiagnosticEngine — 规则调度入口
|
||||
# ---------------------------------------------------------------------------
|
||||
class DiagnosticEngine:
|
||||
"""诊断规则引擎。
|
||||
|
||||
用法::
|
||||
|
||||
engine = DiagnosticEngine(db=SessionLocal(), scanner=scanner,
|
||||
policy_store=policy_store, docker_status="full")
|
||||
results = engine.run_all(run_by="page_request")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db, # SQLAlchemy Session
|
||||
scanner, # Scanner 实例
|
||||
policy_store, # PolicyStore 实例
|
||||
docker_status: str = "none", # full / partial / none
|
||||
docker_log_tail: int = 500,
|
||||
):
|
||||
self.db = db
|
||||
self.scanner = scanner
|
||||
self.policy_store = policy_store
|
||||
self.docker_status = docker_status
|
||||
self.docker_log_tail = docker_log_tail
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 入口方法
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run_all(self, run_by: str = "page_request") -> List[RuleResult]:
|
||||
"""运行全部 15 条规则,返回结果列表。"""
|
||||
# 延迟导入,避免循环依赖
|
||||
from app.diagnostic import rules as _rules
|
||||
|
||||
results: List[RuleResult] = []
|
||||
all_funcs = [
|
||||
_rules.check_r001_error_ratio,
|
||||
_rules.check_r002_clamav_db,
|
||||
_rules.check_r003_yara_mtime,
|
||||
_rules.check_r004_engine_health,
|
||||
_rules.check_r005_no_scans,
|
||||
_rules.check_r006_suspicious_over_malicious,
|
||||
_rules.check_r007_reject_ratio,
|
||||
_rules.check_r008_duplicate_sha,
|
||||
# 以下 7 条需要 Docker 能力
|
||||
_rules.check_r009_clamav_crash,
|
||||
_rules.check_r010_postgres_fatal,
|
||||
_rules.check_r011_redis_oom,
|
||||
_rules.check_r012_no_space,
|
||||
_rules.check_r013_nginx_upstream,
|
||||
_rules.check_r014_api_connection,
|
||||
_rules.check_r015_container_restart,
|
||||
_rules.check_r016_restart_history,
|
||||
]
|
||||
|
||||
for fn in all_funcs:
|
||||
try:
|
||||
results.append(fn(self, run_by=run_by))
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 单条规则失败不影响整体
|
||||
results.append(RuleResult(
|
||||
id=getattr(fn, "_rule_id", "UNKNOWN"),
|
||||
severity="info",
|
||||
message=f"规则执行异常: {e}",
|
||||
skipped=True,
|
||||
skip_reason=str(e),
|
||||
run_by=run_by,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def run_page_request_rules(self) -> List[RuleResult]:
|
||||
"""页面请求时必跑的轻量规则(R001/R002/R003/R004/R005)。"""
|
||||
from app.diagnostic import rules as _rules
|
||||
|
||||
results: List[RuleResult] = []
|
||||
for fn in [
|
||||
_rules.check_r001_error_ratio,
|
||||
_rules.check_r002_clamav_db,
|
||||
_rules.check_r003_yara_mtime,
|
||||
_rules.check_r004_engine_health,
|
||||
_rules.check_r005_no_scans,
|
||||
]:
|
||||
try:
|
||||
results.append(fn(self, run_by="page_request"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
results.append(RuleResult(
|
||||
id=getattr(fn, "_rule_id", "UNKNOWN"),
|
||||
severity="info",
|
||||
message=f"规则执行异常: {e}",
|
||||
skipped=True,
|
||||
skip_reason=str(e),
|
||||
))
|
||||
return results
|
||||
|
||||
def run_timer_error_scan(self) -> List[RuleResult]:
|
||||
"""定时器 A:错误类日志(30s 周期)。"""
|
||||
from app.diagnostic import rules as _rules
|
||||
|
||||
results: List[RuleResult] = []
|
||||
for fn in [
|
||||
_rules.check_r001_error_ratio,
|
||||
_rules.check_r005_no_scans,
|
||||
_rules.check_r009_clamav_crash,
|
||||
_rules.check_r010_postgres_fatal,
|
||||
_rules.check_r011_redis_oom,
|
||||
_rules.check_r012_no_space,
|
||||
_rules.check_r013_nginx_upstream,
|
||||
_rules.check_r014_api_connection,
|
||||
]:
|
||||
try:
|
||||
results.append(fn(self, run_by="timer:error_scan"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
results.append(RuleResult(
|
||||
id=getattr(fn, "_rule_id", "UNKNOWN"),
|
||||
severity="info",
|
||||
message=f"规则执行异常: {e}",
|
||||
skipped=True,
|
||||
skip_reason=str(e),
|
||||
))
|
||||
return results
|
||||
|
||||
def run_timer_db_aggregate(self) -> List[RuleResult]:
|
||||
"""定时器 B:DB 聚合类(60s 周期)。"""
|
||||
from app.diagnostic import rules as _rules
|
||||
|
||||
results: List[RuleResult] = []
|
||||
for fn in [
|
||||
_rules.check_r006_suspicious_over_malicious,
|
||||
_rules.check_r007_reject_ratio,
|
||||
_rules.check_r008_duplicate_sha,
|
||||
]:
|
||||
try:
|
||||
results.append(fn(self, run_by="timer:db_aggregate"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
results.append(RuleResult(
|
||||
id=getattr(fn, "_rule_id", "UNKNOWN"),
|
||||
severity="info",
|
||||
message=f"规则执行异常: {e}",
|
||||
skipped=True,
|
||||
skip_reason=str(e),
|
||||
))
|
||||
return results
|
||||
|
||||
def run_timer_container(self) -> List[RuleResult]:
|
||||
"""定时器 C:容器/引擎类(300s 周期)。"""
|
||||
from app.diagnostic import rules as _rules
|
||||
|
||||
results: List[RuleResult] = []
|
||||
for fn in [
|
||||
_rules.check_r002_clamav_db,
|
||||
_rules.check_r003_yara_mtime,
|
||||
_rules.check_r004_engine_health,
|
||||
_rules.check_r015_container_restart,
|
||||
_rules.check_r016_restart_history,
|
||||
]:
|
||||
try:
|
||||
results.append(fn(self, run_by="timer:container_check"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
results.append(RuleResult(
|
||||
id=getattr(fn, "_rule_id", "UNKNOWN"),
|
||||
severity="info",
|
||||
message=f"规则执行异常: {e}",
|
||||
skipped=True,
|
||||
skip_reason=str(e),
|
||||
))
|
||||
return results
|
||||
878
app/diagnostic/rules.py
Normal file
878
app/diagnostic/rules.py
Normal file
@@ -0,0 +1,878 @@
|
||||
"""诊断规则实现(15 条)。
|
||||
|
||||
规则按类型分四组:
|
||||
页面必跑(轻量): R001 / R002 / R003 / R004 / R005
|
||||
DB 聚合: R006 / R007 / R008
|
||||
日志匹配: R009-R014(需要 docker.sock)
|
||||
容器状态: R015(需要 docker.sock)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
from app.config import get_settings
|
||||
from app.diagnostic.config import get_diag_config
|
||||
from app.diagnostic.engine import DiagnosticEngine, RuleResult
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 模块级状态缓存(用于检测重启事件)
|
||||
# key: container_name
|
||||
# value: {"restart_count": int, "started_at": str, "checked_at": float}
|
||||
# ======================================================================
|
||||
_CONTAINER_LAST_STATE: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _ok(rule_id: str, message: str, hint: str = "",
|
||||
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
|
||||
return RuleResult(
|
||||
id=rule_id, severity="ok", message=message, hint=hint,
|
||||
evidence=evidence or [], run_by=run_by,
|
||||
)
|
||||
|
||||
|
||||
def _warn(rule_id: str, message: str, hint: str = "",
|
||||
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
|
||||
return RuleResult(
|
||||
id=rule_id, severity="medium", message=message, hint=hint,
|
||||
evidence=evidence or [], run_by=run_by,
|
||||
)
|
||||
|
||||
|
||||
def _crit(rule_id: str, message: str, hint: str = "",
|
||||
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
|
||||
return RuleResult(
|
||||
id=rule_id, severity="high", message=message, hint=hint,
|
||||
evidence=evidence or [], run_by=run_by,
|
||||
)
|
||||
|
||||
|
||||
def _skip(rule_id: str, reason: str, run_by: str = "") -> RuleResult:
|
||||
return RuleResult(
|
||||
id=rule_id, severity="info", message=f"规则未运行: {reason}",
|
||||
skipped=True, skip_reason=reason, run_by=run_by,
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 工具函数
|
||||
# ======================================================================
|
||||
|
||||
def _fetch_recent_verdict_counts(engine: DiagnosticEngine, limit: int = 100) -> Dict[str, int]:
|
||||
"""查最近 N 次扫描的 verdict 分布。
|
||||
|
||||
用子查询先 limit 再 group_by,兼容 SQLAlchemy 2.0。
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from app.models import ScanTask
|
||||
|
||||
# 先子查询拿到最近 N 条 verdict
|
||||
sub_stmt = (
|
||||
select(ScanTask.verdict)
|
||||
.order_by(ScanTask.created_at.desc())
|
||||
.limit(limit)
|
||||
).subquery()
|
||||
|
||||
# 再在外层 group_by 统计
|
||||
rows = engine.db.execute(
|
||||
select(sub_stmt.c.verdict, func.count(sub_stmt.c.verdict))
|
||||
.group_by(sub_stmt.c.verdict)
|
||||
).all()
|
||||
|
||||
return {v or "unknown": int(c) for v, c in rows}
|
||||
|
||||
|
||||
def _count_24h(engine: DiagnosticEngine) -> int:
|
||||
from app.models import ScanTask
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=24)
|
||||
return (
|
||||
engine.db.query(func.count(ScanTask.id))
|
||||
.filter(ScanTask.created_at >= cutoff)
|
||||
.scalar() or 0
|
||||
)
|
||||
|
||||
|
||||
def _fetch_recent_tasks(engine: DiagnosticEngine,
|
||||
limit: int = 500) -> List[Any]:
|
||||
from app.models import ScanTask
|
||||
return (
|
||||
engine.db.query(ScanTask)
|
||||
.order_by(ScanTask.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def _docker_logs(container: str, tail: int = 500) -> str:
|
||||
"""用 docker SDK 取容器日志。失败返回空字符串。"""
|
||||
try:
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
client = get_client()
|
||||
if not client:
|
||||
return ""
|
||||
c = client.containers.get(container)
|
||||
raw = c.logs(tail=tail)
|
||||
return raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
|
||||
except Exception: # noqa: BLE001
|
||||
return ""
|
||||
|
||||
|
||||
def _docker_inspect(container: str) -> Optional[Dict[str, Any]]:
|
||||
"""用 docker SDK 取容器状态元数据。失败返回 None。"""
|
||||
try:
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
client = get_client()
|
||||
if not client:
|
||||
return None
|
||||
c = client.containers.get(container)
|
||||
attrs = c.attrs or {}
|
||||
state = attrs.get("State", {})
|
||||
return {
|
||||
"status": state.get("Status", "unknown"),
|
||||
"restart_count": attrs.get("RestartCount", 0),
|
||||
"started_at": state.get("StartedAt", ""),
|
||||
"image": attrs.get("Config", {}).get("Image", ""),
|
||||
}
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def _grep_patterns(text: str, patterns: List[str],
|
||||
max_lines: int = 20) -> List[str]:
|
||||
"""从日志文本中抓取命中 pattern 的行。"""
|
||||
hits: List[str] = []
|
||||
for line in text.splitlines():
|
||||
low = line.lower()
|
||||
for p in patterns:
|
||||
if p.lower() in low:
|
||||
hits.append(line.strip())
|
||||
break
|
||||
if len(hits) >= max_lines:
|
||||
break
|
||||
return hits
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R001: error 比例
|
||||
# ======================================================================
|
||||
|
||||
def check_r001_error_ratio(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
cfg = get_diag_config()
|
||||
counts = _fetch_recent_verdict_counts(engine, 100)
|
||||
total = sum(counts.values())
|
||||
if total == 0:
|
||||
return _ok("R001", "暂无扫描记录", run_by=run_by)
|
||||
|
||||
error_count = counts.get("error", 0)
|
||||
error_ratio = error_count / total
|
||||
threshold = cfg.error_ratio_threshold
|
||||
|
||||
if error_ratio > threshold:
|
||||
return _crit(
|
||||
"R001",
|
||||
f"错误比例偏高({error_ratio:.1%} > 阈值 {threshold:.0%})",
|
||||
f"近 100 次扫描 {error_count} 次 error",
|
||||
hint="建议检查 ClamAV / Redis / Postgres 引擎健康",
|
||||
evidence=[f"近 100 次 verdict 分布: {counts}"],
|
||||
run_by=run_by,
|
||||
)
|
||||
return _ok(
|
||||
"R001",
|
||||
f"错误比例正常({error_ratio:.1%},阈值 {threshold:.0%})",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r001_error_ratio._rule_id = "R001"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R002: ClamAV 病毒库日期
|
||||
# ======================================================================
|
||||
|
||||
def check_r002_clamav_db(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
cfg = get_diag_config()
|
||||
try:
|
||||
clamav_info = engine.scanner.health().get("clamav", {})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return _skip("R002", f"ClamAV 健康检查异常: {e}", run_by=run_by)
|
||||
|
||||
sig_date_str = clamav_info.get("signature_date")
|
||||
if not sig_date_str:
|
||||
available = clamav_info.get("available", clamav_info.get("ok", False))
|
||||
if not available:
|
||||
return _crit(
|
||||
"R002", "ClamAV 引擎不可用",
|
||||
hint="检查 clamd 容器是否在运行",
|
||||
run_by=run_by,
|
||||
)
|
||||
return _warn(
|
||||
"R002", "ClamAV 病毒库日期未知",
|
||||
hint="在线更新失败?U 盘离线导入",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
# signature_date 可能是字符串如 "2026-10-14"
|
||||
try:
|
||||
sig_date = datetime.strptime(str(sig_date_str)[:10], "%Y-%m-%d").date()
|
||||
except Exception: # noqa: BLE001
|
||||
return _skip("R002", f"无法解析病毒库日期: {sig_date_str}", run_by=run_by)
|
||||
|
||||
days_old = (datetime.now().date() - sig_date).days
|
||||
stale = cfg.clamav_db_stale_days
|
||||
|
||||
if days_old > stale:
|
||||
return _warn(
|
||||
"R002",
|
||||
f"ClamAV 病毒库已 {days_old} 天未更新",
|
||||
hint="在线更新不可用 → U 盘离线导入 main.cvd / daily.cvd",
|
||||
evidence=[f"signature_date = {sig_date_str}"],
|
||||
run_by=run_by,
|
||||
)
|
||||
return _ok(
|
||||
"R002",
|
||||
f"ClamAV 病毒库新鲜({sig_date_str},{days_old} 天)",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r002_clamav_db._rule_id = "R002"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R003: YARA 规则 mtime
|
||||
# ======================================================================
|
||||
|
||||
def check_r003_yara_mtime(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
settings = get_settings()
|
||||
rules_dir = settings.rules_dir
|
||||
|
||||
if not os.path.isdir(rules_dir):
|
||||
return _skip("R003", f"规则目录不存在: {rules_dir}", run_by=run_by)
|
||||
|
||||
yamls = [f for f in os.listdir(rules_dir) if f.endswith((".yar", ".yara", ".rules"))]
|
||||
if not yamls:
|
||||
return _warn(
|
||||
"R003", "YARA 规则目录为空",
|
||||
hint="检查 rules_dir 配置是否正确",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
latest_mtime = max(
|
||||
os.path.getmtime(os.path.join(rules_dir, f)) for f in yamls
|
||||
)
|
||||
latest_date = datetime.fromtimestamp(latest_mtime).date()
|
||||
days_old = (datetime.now().date() - latest_date).days
|
||||
|
||||
# 对比策略版本发布时间(如果策略文件比规则新,说明规则可能没更新)
|
||||
policy_file = settings.policy_file
|
||||
policy_mtime = None
|
||||
if os.path.isfile(policy_file):
|
||||
policy_mtime = os.path.getmtime(policy_file)
|
||||
|
||||
evidence = [f"规则目录: {rules_dir}", f"规则文件数: {len(yamls)}",
|
||||
f"最新规则 mtime: {latest_date}"]
|
||||
if policy_mtime and policy_mtime > latest_mtime:
|
||||
evidence.append(f"策略文件比规则新(策略: {datetime.fromtimestamp(policy_mtime).date()})")
|
||||
return _warn(
|
||||
"R003",
|
||||
"策略文件比 YARA 规则新,规则可能未随策略版本更新",
|
||||
hint="同步更新规则文件或确认规则已加载",
|
||||
evidence=evidence,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok(
|
||||
"R003",
|
||||
f"YARA 规则正常({len(yamls)} 条规则,最新 {latest_date},{days_old} 天)",
|
||||
evidence=evidence,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r003_yara_mtime._rule_id = "R003"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R004: 引擎健康
|
||||
# ======================================================================
|
||||
|
||||
def check_r004_engine_health(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
try:
|
||||
health = engine.scanner.health()
|
||||
except Exception as e: # noqa: BLE001
|
||||
return _crit("R004", f"引擎健康检查异常: {e}", run_by=run_by)
|
||||
|
||||
offline: List[str] = []
|
||||
for name, info in health.items():
|
||||
ok = info.get("ok", info.get("available", False))
|
||||
if not ok:
|
||||
offline.append(name)
|
||||
|
||||
if offline:
|
||||
return _crit(
|
||||
"R004",
|
||||
f"以下引擎不可用: {', '.join(offline)}",
|
||||
hint="检查对应容器状态和网络连通",
|
||||
evidence=[f"{n}: {health.get(n, {})}" for n in offline],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
names = list(health.keys())
|
||||
return _ok(
|
||||
"R004",
|
||||
f"所有 {len(names)} 个引擎正常在线",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r004_engine_health._rule_id = "R004"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R005: 24h 无扫描
|
||||
# ======================================================================
|
||||
|
||||
def check_r005_no_scans(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
count = _count_24h(engine)
|
||||
if count == 0:
|
||||
return _warn(
|
||||
"R005", "过去 24 小时内没有扫描记录",
|
||||
hint="摆渡系统可能停了;如果是正常停工可以忽略",
|
||||
run_by=run_by,
|
||||
)
|
||||
return _ok(
|
||||
"R005", f"过去 24 小时 {count} 次扫描",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r005_no_scans._rule_id = "R005"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R006: suspicious > malicious
|
||||
# ======================================================================
|
||||
|
||||
def check_r006_suspicious_over_malicious(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
cfg = get_diag_config()
|
||||
if not cfg.suspicious_exceeds_malicious:
|
||||
return _skip("R006", "规则已在配置中禁用", run_by=run_by)
|
||||
|
||||
counts = _fetch_recent_verdict_counts(engine, 200)
|
||||
suspicious = counts.get("suspicious", 0)
|
||||
malicious = counts.get("malicious", 0)
|
||||
|
||||
if suspicious > malicious and suspicious > 0:
|
||||
return _warn(
|
||||
"R006",
|
||||
f"可疑数({suspicious})超过恶意数({malicious})",
|
||||
hint="suspicious 阈值配置可能过松;建议调紧让更多文件直接判为 malicious",
|
||||
evidence=[f"近 200 次 verdict 分布: {counts}"],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok(
|
||||
"R006",
|
||||
f"suspicious={suspicious} vs malicious={malicious}",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r006_suspicious_over_malicious._rule_id = "R006"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R007: reject 比例
|
||||
# ======================================================================
|
||||
|
||||
def check_r007_reject_ratio(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
cfg = get_diag_config()
|
||||
counts = _fetch_recent_verdict_counts(engine, 200)
|
||||
total = sum(counts.values())
|
||||
if total == 0:
|
||||
return _ok("R007", "暂无扫描记录", run_by=run_by)
|
||||
|
||||
reject = counts.get("reject", 0)
|
||||
ratio = reject / total
|
||||
threshold = cfg.reject_ratio_threshold
|
||||
|
||||
if ratio > threshold:
|
||||
return _warn(
|
||||
"R007",
|
||||
f"策略拒绝比例偏高({ratio:.1%} > 阈值 {threshold:.0%})",
|
||||
hint="检查拒绝原因 TOP;如果只是正常扩展名策略可以调宽",
|
||||
evidence=[f"近 200 次 verdict 分布: {counts}"],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok(
|
||||
"R007",
|
||||
f"策略拒绝比例正常({ratio:.1%})",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r007_reject_ratio._rule_id = "R007"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R008: 同 SHA256 重复扫描
|
||||
# ======================================================================
|
||||
|
||||
def check_r008_duplicate_sha(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
from app.models import ScanTask
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
rows = (
|
||||
engine.db.query(ScanTask.sha256, func.count(ScanTask.id))
|
||||
.filter(ScanTask.created_at >= cutoff)
|
||||
.group_by(ScanTask.sha256)
|
||||
.having(func.count(ScanTask.id) > 1)
|
||||
.all()
|
||||
)
|
||||
|
||||
if rows:
|
||||
dups = [(sha[:12], c) for sha, c in rows]
|
||||
return _crit(
|
||||
"R008",
|
||||
f"近 1 小时有 {len(rows)} 个 SHA256 被重复扫描",
|
||||
hint="幂等机制可能失效;检查 idempotency_records 表写入",
|
||||
evidence=[f"sha256={sha} count={c}" for sha, c in dups[:10]],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok("R008", "幂等机制正常(无重复 SHA256)", run_by=run_by)
|
||||
|
||||
check_r008_duplicate_sha._rule_id = "R008"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 日志匹配型规则(需要 docker.sock)
|
||||
# ======================================================================
|
||||
|
||||
def _check_docker_available(engine: DiagnosticEngine) -> Optional[RuleResult]:
|
||||
"""如果 Docker 不可用,返回 skip RuleResult;否则返回 None(继续执行)。"""
|
||||
if engine.docker_status == "none":
|
||||
return _skip(None, "Docker 能力未启用", run_by="")
|
||||
return None
|
||||
|
||||
|
||||
def check_r009_clamav_crash(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R009"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
logs = _docker_logs("meshsafe-clamav", cfg.docker_log_tail)
|
||||
patterns = ["Killed", "exit 137", "segfault", "oom-killed", "Out of memory"]
|
||||
hits = _grep_patterns(logs, patterns)
|
||||
|
||||
# 同时检查 docker inspect 的 restart_count
|
||||
info = _docker_inspect("meshsafe-clamav")
|
||||
restart_count = info.get("restart_count", 0) if info else 0
|
||||
|
||||
if hits or restart_count > 3:
|
||||
ev = hits[:5] if hits else []
|
||||
if restart_count > 3:
|
||||
ev.append(f"docker inspect restart_count={restart_count}")
|
||||
return _crit(
|
||||
"R009",
|
||||
f"ClamAV 可能崩溃(restart_count={restart_count},日志命中 {len(hits)} 行)",
|
||||
hint="OOM kill → 给 clamav 容器加内存限制 512m → 1g;\nsegfault → 检查 ClamAV 版本兼容性",
|
||||
evidence=ev,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok(
|
||||
"R009",
|
||||
f"ClamAV 容器稳定(restart_count={restart_count},无崩溃日志)",
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r009_clamav_crash._rule_id = "R009"
|
||||
|
||||
|
||||
def check_r010_postgres_fatal(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R010"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
logs = _docker_logs("meshsafe-postgres", cfg.docker_log_tail)
|
||||
patterns = ["FATAL", "PANIC", "too many connections", "database is in recovery"]
|
||||
hits = _grep_patterns(logs, patterns)
|
||||
|
||||
if hits:
|
||||
return _crit(
|
||||
"R010",
|
||||
f"Postgres 出现严重错误(命中 {len(hits)} 行)",
|
||||
hint="too many connections → 调大 max_connections;\nFATAL → 检查磁盘空间和 WAL",
|
||||
evidence=hits[:8],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok("R010", "Postgres 日志无 FATAL/PANIC", run_by=run_by)
|
||||
|
||||
check_r010_postgres_fatal._rule_id = "R010"
|
||||
|
||||
|
||||
def check_r011_redis_oom(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R011"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
logs = _docker_logs("meshsafe-redis", cfg.docker_log_tail)
|
||||
# 注意:不能用 "LOADING" — Redis 正常启动也会打 "Loading RDB / Done loading"
|
||||
# 只匹配真正有问题的:加载卡死循环、OOM、panic、assertion
|
||||
patterns = [
|
||||
"OOM command not allowed",
|
||||
"out of memory",
|
||||
"LOADING loop",
|
||||
"still loading",
|
||||
"LOADING state",
|
||||
"panic",
|
||||
"Assertion",
|
||||
]
|
||||
hits = _grep_patterns(logs, patterns)
|
||||
|
||||
if hits:
|
||||
return _crit(
|
||||
"R011",
|
||||
f"Redis 可能 OOM 或 panic(命中 {len(hits)} 行)",
|
||||
hint="OOM → 调大 maxmemory 或清缓存;\nLOADING 循环 → 检查 RDB/AOF 恢复",
|
||||
evidence=hits[:5],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok("R011", "Redis 日志无 OOM/panic", run_by=run_by)
|
||||
|
||||
check_r011_redis_oom._rule_id = "R011"
|
||||
|
||||
|
||||
def check_r012_no_space(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R012"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
all_patterns = ["No space left on device", "ENOSPC", "disk quota exceeded"]
|
||||
containers = [
|
||||
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
||||
"meshsafe-redis", "meshsafe-nginx",
|
||||
]
|
||||
|
||||
all_hits: List[str] = []
|
||||
for c in containers:
|
||||
logs = _docker_logs(c, cfg.docker_log_tail)
|
||||
hits = _grep_patterns(logs, all_patterns, max_lines=5)
|
||||
for h in hits:
|
||||
all_hits.append(f"[{c}] {h}")
|
||||
if len(all_hits) >= 10:
|
||||
break
|
||||
|
||||
if all_hits:
|
||||
return _crit(
|
||||
"R012",
|
||||
"容器日志出现磁盘空间不足(严重!)",
|
||||
hint="立即清理磁盘,否则扫描会持续失败",
|
||||
evidence=all_hits[:10],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok("R012", "所有容器日志无磁盘空间错误", run_by=run_by)
|
||||
|
||||
check_r012_no_space._rule_id = "R012"
|
||||
|
||||
|
||||
def check_r013_nginx_upstream(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R013"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
logs = _docker_logs("meshsafe-nginx", cfg.docker_log_tail)
|
||||
# 只匹配真正的 upstream 故障。
|
||||
# 注意 nginx 正常日志里有 "upstream response is buffered to a temporary file",
|
||||
# 那是响应太大被缓冲的正常行为,绝不能在这里误伤。
|
||||
patterns = [
|
||||
"Connection refused",
|
||||
"upstream timed out",
|
||||
"upstream prematurely closed",
|
||||
"no live upstreams",
|
||||
"connect() failed",
|
||||
"502 Bad Gateway",
|
||||
"upstream server temporarily disabled",
|
||||
]
|
||||
hits = _grep_patterns(logs, patterns)
|
||||
|
||||
if hits:
|
||||
return _warn(
|
||||
"R013",
|
||||
f"Nginx upstream 异常(命中 {len(hits)} 行)",
|
||||
hint="api 容器可能临时不可用;检查 api 健康状态",
|
||||
evidence=hits[:5],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok("R013", "Nginx 无 upstream 异常", run_by=run_by)
|
||||
|
||||
check_r013_nginx_upstream._rule_id = "R013"
|
||||
|
||||
|
||||
def check_r014_api_connection(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R014"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
logs = _docker_logs("meshsafe-api", cfg.docker_log_tail)
|
||||
patterns = ["Redis connection refused", "Postgres connection refused",
|
||||
"ConnectionError", "OperationalError", "can't connect"]
|
||||
hits = _grep_patterns(logs, patterns)
|
||||
|
||||
if hits:
|
||||
return _crit(
|
||||
"R014",
|
||||
f"API 与依赖服务连接异常(命中 {len(hits)} 行)",
|
||||
hint="Redis/Postgres 网络断了?检查 docker 网络和依赖容器健康",
|
||||
evidence=hits[:5],
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok("R014", "API 与依赖连接正常", run_by=run_by)
|
||||
|
||||
check_r014_api_connection._rule_id = "R014"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R015: 容器重启次数
|
||||
# ======================================================================
|
||||
|
||||
def check_r015_container_restart(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R015"
|
||||
return skip
|
||||
|
||||
containers = [
|
||||
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
||||
"meshsafe-redis", "meshsafe-nginx",
|
||||
]
|
||||
|
||||
now_ts = time.time()
|
||||
bj = datetime.now(timezone(timedelta(hours=8)))
|
||||
restart_events_now: List[str] = [] # 本轮检测到的新事件(写进 evidence)
|
||||
new_restart_containers: List[str] = [] # 有新重启事件的容器名
|
||||
|
||||
for c in containers:
|
||||
info = _docker_inspect(c)
|
||||
if not info:
|
||||
continue
|
||||
|
||||
cur_restart = info.get("restart_count", 0)
|
||||
cur_started = info.get("started_at", "") or ""
|
||||
|
||||
old = _CONTAINER_LAST_STATE.get(c)
|
||||
restart_changed = False
|
||||
started_changed = False
|
||||
|
||||
if old is not None:
|
||||
# 第一次跑没有 old,不算事件,只建缓存
|
||||
restart_changed = cur_restart > old.get("restart_count", -1)
|
||||
started_changed = cur_started != old.get("started_at", "")
|
||||
|
||||
if restart_changed or started_changed:
|
||||
trigger = "restart_count_increased" if restart_changed else "started_at_changed"
|
||||
uptime_before_s = 0
|
||||
try:
|
||||
uptime_before_s = int(now_ts - old.get("checked_at", now_ts))
|
||||
except Exception:
|
||||
uptime_before_s = 0
|
||||
m, uptime_before_s = divmod(uptime_before_s, 60)
|
||||
h, m = divmod(m, 60)
|
||||
uptime_before_str = (
|
||||
f"{h}h {m}m" if h > 0 else f"{m}m" if m > 0 else "刚启动"
|
||||
)
|
||||
|
||||
event_line = (
|
||||
f"{bj.strftime('%H:%M:%S')} {c} "
|
||||
f"restart_count {old.get('restart_count','?')} → {cur_restart} "
|
||||
f"uptime_before={uptime_before_str} "
|
||||
f"trigger={trigger}"
|
||||
)
|
||||
restart_events_now.append(event_line)
|
||||
new_restart_containers.append(c)
|
||||
|
||||
# 持久化重启事件
|
||||
try:
|
||||
from app.diagnostic.storage import (
|
||||
append_restart_event, snapshot_on_restart,
|
||||
)
|
||||
append_restart_event(
|
||||
container=c,
|
||||
restart_before=old.get("restart_count", 0),
|
||||
restart_after=cur_restart,
|
||||
uptime_before=uptime_before_str,
|
||||
uptime_before_seconds=now_ts - old.get("checked_at", now_ts),
|
||||
trigger=trigger,
|
||||
)
|
||||
# 抓重启时的日志片段
|
||||
try:
|
||||
snapshot_on_restart(c, tail=300)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 更新缓存为当前状态
|
||||
_CONTAINER_LAST_STATE[c] = {
|
||||
"restart_count": cur_restart,
|
||||
"started_at": cur_started,
|
||||
"checked_at": now_ts,
|
||||
}
|
||||
|
||||
# ---------- 判定 severity ----------
|
||||
# 1. 有新重启事件 → medium(哪怕 restart_count 还没超 3)
|
||||
# 2. restart_count > 3 → crit
|
||||
# 3. 否则 → ok
|
||||
crit_list: List[str] = []
|
||||
all_status: List[str] = []
|
||||
for c in containers:
|
||||
info = _docker_inspect(c)
|
||||
if info:
|
||||
r = info.get("restart_count", 0)
|
||||
all_status.append(f"{c}: restart={r}")
|
||||
if r > 3:
|
||||
crit_list.append(f"{c} (restart_count={r})")
|
||||
|
||||
if crit_list:
|
||||
# 严重:累积重启次数过多
|
||||
combined_evidence = list(restart_events_now) + [f"⚠️ {x}" for x in crit_list]
|
||||
return _crit(
|
||||
"R015",
|
||||
f"以下容器不稳定(restart_count > 3): {', '.join(crit_list)}",
|
||||
hint="检查对应容器日志找 crash 根因(OOM/segmentation/配置错误)",
|
||||
evidence=combined_evidence + all_status,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
if new_restart_containers:
|
||||
# 中等:本轮检测到新重启事件(但累积次数还没超 3)
|
||||
return _warn(
|
||||
"R015",
|
||||
f"本轮检测到容器重启事件: {', '.join(new_restart_containers)}",
|
||||
hint="开发/运维可能执行了 docker restart,或容器健康检查触发了自动重启。"
|
||||
"日志片段已自动归档到 diagnostics_logs/*-restart-*.log",
|
||||
evidence=restart_events_now + all_status,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
return _ok(
|
||||
"R015",
|
||||
"所有容器稳定(restart_count 均 ≤ 3,本轮无新重启事件)",
|
||||
evidence=all_status,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r015_container_restart._rule_id = "R015"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# R016: 容器重启历史汇总(读持久化 restart_events.jsonl)
|
||||
# ======================================================================
|
||||
|
||||
def check_r016_restart_history(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
||||
"""读 restart_events.jsonl + compose_state,给出过去 N 天每个容器重启次数汇总。
|
||||
|
||||
和 R015 的区别:
|
||||
R015 = 当前状态检测(restart_count 阈值 + 本轮变化)
|
||||
R016 = 历史回顾(持久化事件日志聚合)
|
||||
"""
|
||||
skip = _check_docker_available(engine)
|
||||
if skip:
|
||||
skip.id = "R016"
|
||||
return skip
|
||||
|
||||
cfg = get_diag_config()
|
||||
|
||||
# 1. 读持久化重启事件
|
||||
events: List[Dict[str, Any]] = []
|
||||
try:
|
||||
from app.diagnostic.storage import load_restart_events
|
||||
if cfg.storage_enabled:
|
||||
events = load_restart_events(cfg.history_retention_days)
|
||||
except Exception:
|
||||
events = []
|
||||
|
||||
# 2. 按容器聚合
|
||||
by_container: Dict[str, List[Dict[str, Any]]] = {}
|
||||
for ev in events:
|
||||
c = ev.get("container", "?")
|
||||
by_container.setdefault(c, []).append(ev)
|
||||
|
||||
# 3. 取当前 restart_count(compose_state)
|
||||
containers = [
|
||||
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
||||
"meshsafe-redis", "meshsafe-nginx",
|
||||
]
|
||||
current_counts: Dict[str, int] = {}
|
||||
for c in containers:
|
||||
info = _docker_inspect(c)
|
||||
if info:
|
||||
current_counts[c] = info.get("restart_count", 0)
|
||||
|
||||
# 4. 构建 evidence(每个容器一行汇总)
|
||||
evidence: List[str] = []
|
||||
total_events = 0
|
||||
problem_containers: List[str] = []
|
||||
for c in containers:
|
||||
hist_count = len(by_container.get(c, []))
|
||||
cur_count = current_counts.get(c, 0)
|
||||
total_events += hist_count
|
||||
|
||||
line = f"{c}: 历史重启事件={hist_count} 当前restart_count={cur_count}"
|
||||
if hist_count > 0:
|
||||
last_ev = by_container[c][-1]
|
||||
line += f" 最近事件={last_ev.get('ts')}"
|
||||
evidence.append(line)
|
||||
|
||||
if hist_count >= 3:
|
||||
problem_containers.append(f"{c}({hist_count}次)")
|
||||
|
||||
# 5. 判定
|
||||
if total_events == 0:
|
||||
return _ok(
|
||||
"R016",
|
||||
f"过去 {cfg.history_retention_days} 天内,无容器重启事件记录",
|
||||
evidence=evidence,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
if problem_containers:
|
||||
return _warn(
|
||||
"R016",
|
||||
f"过去 {cfg.history_retention_days} 天内,以下容器重启较频繁: "
|
||||
f"{', '.join(problem_containers)}",
|
||||
hint="开发/运维可能执行了多次 docker restart,"
|
||||
"或容器健康检查/初始化阶段触发重启。"
|
||||
"详见 restart_events.json 和 diagnostics_logs/*-restart-*.log",
|
||||
evidence=evidence,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
# 有事件但不频繁
|
||||
return _ok(
|
||||
"R016",
|
||||
f"过去 {cfg.history_retention_days} 天内共 {total_events} 次重启事件(均不频繁)",
|
||||
evidence=evidence,
|
||||
run_by=run_by,
|
||||
)
|
||||
|
||||
check_r016_restart_history._rule_id = "R016"
|
||||
268
app/diagnostic/scheduler.py
Normal file
268
app/diagnostic/scheduler.py
Normal file
@@ -0,0 +1,268 @@
|
||||
"""三层定时器循环。
|
||||
|
||||
在 FastAPI startup 事件中用 asyncio.create_task() 启动。
|
||||
定时器在后台跑,每次迭代独立 try/except + timeout,不会卡死主 event loop。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.diagnostic.config import DiagConfig, get_diag_config
|
||||
from app.diagnostic.engine import DiagnosticEngine, RuleResult
|
||||
from app.services.scanner import Scanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# 诊断套件全局运行时状态(单例 dict)
|
||||
_DIAG_STATE: Dict[str, Any] = {
|
||||
"enabled": False,
|
||||
"docker_status": "none",
|
||||
"docker_info": {},
|
||||
"started_at": None,
|
||||
"rules_cache": {}, # {rule_id: {result, ts, run_by}}
|
||||
"latest": {"rules": [], "generated_at": None, "docker_status": "none"},
|
||||
}
|
||||
|
||||
# 后台引用(防止 GC 收走)
|
||||
_TIMER_TASKS: List[asyncio.Task] = []
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 外部访问入口
|
||||
# ======================================================================
|
||||
|
||||
def get_diag_state() -> Dict[str, Any]:
|
||||
"""获取诊断套件运行时状态(给页面/API 查询用)。"""
|
||||
cache = {k: {"result": v["result"].to_dict() if isinstance(v.get("result"), RuleResult) else v.get("result"),
|
||||
"ts": v.get("ts"), "run_by": v.get("run_by")}
|
||||
for k, v in _DIAG_STATE["rules_cache"].items()}
|
||||
return {
|
||||
"enabled": _DIAG_STATE["enabled"],
|
||||
"docker_status": _DIAG_STATE["docker_status"],
|
||||
"docker_info": _DIAG_STATE["docker_info"],
|
||||
"started_at": _DIAG_STATE["started_at"],
|
||||
"rules_cache": cache,
|
||||
"latest": _DIAG_STATE["latest"],
|
||||
}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 后台入口(在 app.main.startup 中调用)
|
||||
# ======================================================================
|
||||
|
||||
def start_timers(docker_status: str, docker_info: Dict[str, Any]) -> None:
|
||||
"""启动三个定时器循环(后台 daemon task)。"""
|
||||
cfg = get_diag_config()
|
||||
_DIAG_STATE["enabled"] = cfg.enabled
|
||||
_DIAG_STATE["docker_status"] = docker_status
|
||||
_DIAG_STATE["docker_info"] = docker_info
|
||||
_DIAG_STATE["started_at"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
if not cfg.enabled:
|
||||
logger.info("MeshSafe Diagnostics: disabled")
|
||||
return
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if cfg.error_scan_interval > 0:
|
||||
_TIMER_TASKS.append(loop.create_task(
|
||||
_timer_loop("error_scan", cfg.error_scan_interval,
|
||||
lambda eng: eng.run_timer_error_scan(),
|
||||
docker_status, cfg.docker_log_tail),
|
||||
name="diag-error-scan",
|
||||
))
|
||||
logger.info("诊断定时器启动: error_scan (%ss)", cfg.error_scan_interval)
|
||||
|
||||
if cfg.db_aggregate_interval > 0:
|
||||
_TIMER_TASKS.append(loop.create_task(
|
||||
_timer_loop("db_aggregate", cfg.db_aggregate_interval,
|
||||
lambda eng: eng.run_timer_db_aggregate(),
|
||||
docker_status, cfg.docker_log_tail),
|
||||
name="diag-db-aggregate",
|
||||
))
|
||||
logger.info("诊断定时器启动: db_aggregate (%ss)", cfg.db_aggregate_interval)
|
||||
|
||||
if cfg.container_check_interval > 0:
|
||||
_TIMER_TASKS.append(loop.create_task(
|
||||
_timer_loop("container_check", cfg.container_check_interval,
|
||||
lambda eng: eng.run_timer_container(),
|
||||
docker_status, cfg.docker_log_tail),
|
||||
name="diag-container-check",
|
||||
))
|
||||
logger.info("诊断定时器启动: container_check (%ss)", cfg.container_check_interval)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 定时器循环(async)
|
||||
# ======================================================================
|
||||
|
||||
async def _timer_loop(
|
||||
name: str,
|
||||
interval: int,
|
||||
runner_fn, # 接受 DiagnosticEngine 返回 List[RuleResult]
|
||||
docker_status: str,
|
||||
docker_log_tail: int,
|
||||
) -> None:
|
||||
"""通用定时器循环。每次迭代独立 try/except + 超时保护。"""
|
||||
|
||||
# 等一个 interval 再开始第一次(避免 startup 时 DB 还没 ready)
|
||||
await asyncio.sleep(min(interval, 15))
|
||||
|
||||
while True:
|
||||
try:
|
||||
# 分布式锁(时间槽位):多 worker 部署时,每个时间槽只有
|
||||
# 一个 worker 能拿到锁并执行,保证执行频率精确等于 interval。
|
||||
if not _acquire_timer_lock(name, interval):
|
||||
logger.debug("诊断定时器 %s 本轮跳过(其他 worker 持锁)", name)
|
||||
await asyncio.sleep(interval)
|
||||
continue
|
||||
|
||||
results = await asyncio.wait_for(
|
||||
asyncio.get_event_loop().run_in_executor(
|
||||
None,
|
||||
lambda: _run_sync(name, runner_fn, docker_status, docker_log_tail),
|
||||
),
|
||||
timeout=30.0,
|
||||
)
|
||||
_cache_results(name, results, docker_status)
|
||||
logger.debug("诊断定时器 %s 跑完: %d 条规则", name, len(results))
|
||||
except asyncio.CancelledError:
|
||||
logger.info("诊断定时器 %s 被取消", name)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("诊断定时器 %s 超时(30s),本轮跳过", name)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("诊断定时器 %s 异常: %s,%ss 后重试", name, e, interval)
|
||||
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 定时器分布式锁(Redis)
|
||||
#
|
||||
# 多 worker 部署(UVICORN_WORKERS > 1)时,每个 worker 都会启动定时器,
|
||||
# 导致:同一轮规则被跑 N 次、历史数据量 ×N、并发写 JSONL 造成文件损坏。
|
||||
# 用 Redis SET NX 抢锁,保证同一时刻只有一个 worker 在跑。
|
||||
# ======================================================================
|
||||
|
||||
_TIMER_LOCK_PREFIX = "meshsafe:diag:timer_lock:"
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _lock_redis():
|
||||
"""定时器锁专用 Redis 客户端(进程内复用,避免每次调用重建连接)。"""
|
||||
import redis # type: ignore
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
return redis.from_url(
|
||||
get_settings().redis_url, decode_responses=True, socket_timeout=3
|
||||
)
|
||||
|
||||
|
||||
def _acquire_timer_lock(timer_name: str, interval: int) -> bool:
|
||||
"""尝试获取定时器锁(时间槽位设计)。
|
||||
|
||||
key 里带上当前时间槽位号:``meshsafe:diag:timer_lock:<timer>:<slot>``,
|
||||
slot = 当前时间 // interval。这样无论多少 worker、谁先谁后,
|
||||
同一个槽位内最多只有一个 worker 能拿到锁并执行,
|
||||
下一个槽位自然放行 —— 执行频率精确等于 interval,不多不少。
|
||||
|
||||
Redis 不可用时返回 True(降级为单 worker 行为,不阻塞主流程)。
|
||||
"""
|
||||
try:
|
||||
client = _lock_redis()
|
||||
slot = int(time.time() // interval)
|
||||
key = f"{_TIMER_LOCK_PREFIX}{timer_name}:{slot}"
|
||||
# EX 只做兜底(槽位 key 不会互相干扰),设 2 倍 interval 足够
|
||||
return bool(client.set(key, "1", nx=True, ex=interval * 2))
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Redis 不可用 → 不加锁(单机单 worker 场景行为不变)
|
||||
logger.debug("定时器锁获取失败,降级为无锁执行: %s", e)
|
||||
return True
|
||||
|
||||
|
||||
def _run_sync(
|
||||
name: str,
|
||||
runner_fn,
|
||||
docker_status: str,
|
||||
docker_log_tail: int,
|
||||
) -> List[RuleResult]:
|
||||
"""在线程池里同步跑规则(避免阻塞 event loop)。"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# 延迟 import 避免循环依赖
|
||||
from app.main import scanner, policy_store
|
||||
eng = DiagnosticEngine(
|
||||
db=db, scanner=scanner, policy_store=policy_store,
|
||||
docker_status=docker_status, docker_log_tail=docker_log_tail,
|
||||
)
|
||||
results = runner_fn(eng)
|
||||
return results
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _cache_results(
|
||||
timer_name: str,
|
||||
results: List[RuleResult],
|
||||
docker_status: str,
|
||||
) -> None:
|
||||
"""把本轮规则结果写入内存缓存(dict)+ 持久化存储。"""
|
||||
now = time.time()
|
||||
for r in results:
|
||||
_DIAG_STATE["rules_cache"][r.id] = {
|
||||
"result": r,
|
||||
"ts": now,
|
||||
"run_by": r.run_by or f"timer:{timer_name}",
|
||||
}
|
||||
|
||||
# 构建 latest 快照(合并所有缓存 + 本轮结果)
|
||||
all_results = []
|
||||
seen = set()
|
||||
for rid, entry in _DIAG_STATE["rules_cache"].items():
|
||||
if rid not in seen and entry.get("result"):
|
||||
all_results.append(entry["result"])
|
||||
seen.add(rid)
|
||||
|
||||
_DIAG_STATE["latest"] = {
|
||||
"rules": sorted(all_results, key=lambda x: x.id),
|
||||
"generated_at": now,
|
||||
"docker_status": docker_status,
|
||||
}
|
||||
|
||||
# ---------- 持久化 ----------
|
||||
try:
|
||||
from app.diagnostic.config import get_diag_config
|
||||
from app.diagnostic.storage import append_rule_results
|
||||
|
||||
cfg = get_diag_config()
|
||||
if cfg.storage_enabled:
|
||||
append_rule_results(results, timer_name)
|
||||
|
||||
# 可选:定时器也存 docker 日志快照(省磁盘所以用较小 tail)
|
||||
if cfg.log_snapshot_on_timer and docker_status != "none":
|
||||
from app.diagnostic.storage import snapshot_container_logs
|
||||
containers = [
|
||||
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
||||
"meshsafe-redis", "meshsafe-nginx",
|
||||
]
|
||||
# 不要每次都存 — 只在 container_check(5 分钟周期)里存一次
|
||||
if timer_name == "container_check":
|
||||
snapshot_container_logs(
|
||||
containers, cfg.log_snapshot_tail,
|
||||
filename_prefix="timer",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("诊断持久化跳过(非致命): %s", e)
|
||||
455
app/diagnostic/storage.py
Normal file
455
app/diagnostic/storage.py
Normal file
@@ -0,0 +1,455 @@
|
||||
"""诊断套件持久化存储模块。
|
||||
|
||||
职责:
|
||||
- 定时器规则结果 → JSON Lines 追加(rules_history.jsonl)
|
||||
- Docker 日志快照 → 存文件(logs/*.log)
|
||||
- 自动清理过期数据(基于配置的 history_retention_days)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.diagnostic.config import get_diag_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _storage_dir() -> Path:
|
||||
"""获取存储根目录。"""
|
||||
cfg = get_diag_config()
|
||||
p = Path(cfg.storage_dir)
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _logs_dir() -> Path:
|
||||
p = _storage_dir() / "logs"
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _rules_history_file() -> Path:
|
||||
return _storage_dir() / "rules_history.jsonl"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 跨进程文件锁写入
|
||||
#
|
||||
# 多 worker 部署时,每个 worker 都会跑诊断定时器。"a" 追加模式在多进程
|
||||
# 下不安全——各进程独立维护文件偏移量,后写的会覆盖先写的内容,把一行
|
||||
# JSON 从中间劈开,产出非法 UTF-8 字节,导致后续读取全部报错。
|
||||
# 这里用 fcntl 文件锁保证同一时刻只有一个进程在写。
|
||||
# ======================================================================
|
||||
|
||||
try:
|
||||
import fcntl # type: ignore
|
||||
|
||||
_HAS_FCNTL = True
|
||||
except ImportError: # pragma: no cover - Windows 平台
|
||||
fcntl = None # type: ignore
|
||||
_HAS_FCNTL = False
|
||||
|
||||
|
||||
def _append_line_locked(path: Path, line: str) -> None:
|
||||
"""向文件追加一行,跨进程互斥。
|
||||
|
||||
Linux 下用 fcntl.flock 排他锁;无 fcntl 时退化为进程内线程锁。
|
||||
"""
|
||||
payload = line + "\n"
|
||||
with open(path, "a", encoding="utf-8") as fp:
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
fp.seek(0, os.SEEK_END)
|
||||
fp.write(payload)
|
||||
fp.flush()
|
||||
os.fsync(fp.fileno())
|
||||
finally:
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _rewrite_file_locked(path: Path, lines: List[str]) -> None:
|
||||
"""整文件重写(清理过期数据用),同样走文件锁。
|
||||
|
||||
写临时文件再原子替换,避免重写过程中其他进程读到半截内容。
|
||||
"""
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as fp:
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
for line in lines:
|
||||
fp.write(line + "\n")
|
||||
fp.flush()
|
||||
os.fsync(fp.fileno())
|
||||
finally:
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 规则历史:JSON Lines 追加
|
||||
# ======================================================================
|
||||
|
||||
def append_rule_results(
|
||||
rules: List[Any],
|
||||
timer_name: str,
|
||||
) -> None:
|
||||
"""定时器每次跑完追加一行 JSONL。
|
||||
|
||||
每行格式::
|
||||
|
||||
{"ts": "2026-10-28T13:12:14", "timer": "error_scan",
|
||||
"docker_status": "full",
|
||||
"rules": [{"id":"R001","severity":"ok","message":"...","run_at":"...","run_by":"timer:error_scan",...}]}
|
||||
"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return
|
||||
|
||||
try:
|
||||
# 北京时间
|
||||
bj = timezone(timedelta(hours=8))
|
||||
entry: Dict[str, Any] = {
|
||||
"ts": datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"timer": timer_name,
|
||||
"docker_status": None,
|
||||
"rules": [
|
||||
r.to_dict() if hasattr(r, "to_dict") else dict(r)
|
||||
for r in rules
|
||||
],
|
||||
}
|
||||
|
||||
f = _rules_history_file()
|
||||
_append_line_locked(f, json.dumps(entry, ensure_ascii=False))
|
||||
|
||||
# 顺带清理过期
|
||||
_trim_history()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("诊断持久化写入失败: %s", e)
|
||||
|
||||
|
||||
def load_rule_history(days: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
"""读取规则历史 JSONL,返回最近 N 天的行。"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return []
|
||||
days = days or cfg.history_retention_days
|
||||
|
||||
f = _rules_history_file()
|
||||
if not f.exists():
|
||||
return []
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
results: List[Dict[str, Any]] = []
|
||||
try:
|
||||
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
||||
for line in fp:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
# 按 ts 判断是否过期
|
||||
ts_str = obj.get("ts", "")
|
||||
try:
|
||||
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
||||
if ts_dt >= cutoff:
|
||||
results.append(obj)
|
||||
except ValueError:
|
||||
# 时间格式不对就不过滤,保留
|
||||
results.append(obj)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("诊断历史读取失败: %s", e)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _trim_history() -> None:
|
||||
"""清理过期的规则历史行(保留文件头)。"""
|
||||
cfg = get_diag_config()
|
||||
days = cfg.history_retention_days
|
||||
|
||||
f = _rules_history_file()
|
||||
if not f.exists():
|
||||
return
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
kept: List[str] = []
|
||||
try:
|
||||
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
||||
total = 0
|
||||
for line in fp:
|
||||
total += 1
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
ts_str = obj.get("ts", "")
|
||||
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
||||
if ts_dt >= cutoff:
|
||||
kept.append(line)
|
||||
except Exception:
|
||||
kept.append(line) # 格式异常的保留
|
||||
|
||||
# 如果删了一些(不是全部),就重写
|
||||
if kept and len(kept) < total:
|
||||
_rewrite_file_locked(f, kept)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("诊断历史清理失败: %s", e)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Docker 日志快照
|
||||
# ======================================================================
|
||||
|
||||
def snapshot_container_logs(
|
||||
containers: List[str],
|
||||
tail: int,
|
||||
filename_prefix: Optional[str] = None,
|
||||
) -> List[Path]:
|
||||
"""把各容器当前 docker logs --tail N 存一份文件。
|
||||
|
||||
文件名: {container}-{prefix}{timestamp}.log
|
||||
prefix 可选(如 timer 存的加前缀 timer-)
|
||||
返回写入的文件列表。
|
||||
"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return []
|
||||
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return []
|
||||
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
prefix = f"{filename_prefix}-" if filename_prefix else ""
|
||||
out: List[Path] = []
|
||||
|
||||
for name in containers:
|
||||
try:
|
||||
c = client.containers.get(name)
|
||||
raw = c.logs(tail=tail)
|
||||
text = (
|
||||
raw.decode("utf-8", errors="replace")
|
||||
if isinstance(raw, (bytes, bytearray))
|
||||
else str(raw)
|
||||
)
|
||||
fname = f"{name}-{prefix}{ts}.log"
|
||||
fpath = _logs_dir() / fname
|
||||
fpath.write_text(text, encoding="utf-8")
|
||||
out.append(fpath)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("容器 %s 日志快照失败: %s", name, e)
|
||||
|
||||
# 顺带清理过期日志
|
||||
_trim_logs()
|
||||
return out
|
||||
|
||||
|
||||
def load_log_snapshots(containers: Optional[List[str]] = None) -> Dict[str, List[Path]]:
|
||||
"""读 logs/ 目录,返回 {container_name: [path, path, ...]}。"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return {}
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=cfg.history_retention_days)
|
||||
result: Dict[str, List[Path]] = {}
|
||||
for f in _logs_dir().glob("*.log"):
|
||||
stem = f.stem # 去掉 .log
|
||||
# 文件名格式: meshsafe-api-timer-20261028-1312 或 meshsafe-api-20261028-1312
|
||||
# 容器名是 stem 里第一个 - 前面的部分... 其实容器名本身也可能带 -
|
||||
# 简化:直接用 stem 整体当 key(导出时按需匹配)
|
||||
# 但我们需要按容器分组,所以用 glob 前缀匹配
|
||||
if containers:
|
||||
matched = None
|
||||
for c in containers:
|
||||
if stem.startswith(c + "-"):
|
||||
matched = c
|
||||
break
|
||||
if matched is None:
|
||||
continue
|
||||
container_key = matched
|
||||
else:
|
||||
# 没指定容器,尝试从文件名推断
|
||||
parts = stem.split("-")
|
||||
container_key = parts[0] if parts else stem
|
||||
|
||||
try:
|
||||
mtime = datetime.fromtimestamp(f.stat().st_mtime)
|
||||
if mtime >= cutoff:
|
||||
result.setdefault(container_key, []).append(f)
|
||||
except Exception:
|
||||
result.setdefault(container_key, []).append(f)
|
||||
|
||||
# 按 mtime 排序(新的在前)
|
||||
for k in result:
|
||||
result[k].sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _trim_logs() -> None:
|
||||
"""清理过期的日志快照文件。"""
|
||||
cfg = get_diag_config()
|
||||
cutoff = datetime.now() - timedelta(days=cfg.history_retention_days)
|
||||
try:
|
||||
for f in _logs_dir().glob("*.log"):
|
||||
try:
|
||||
mtime = datetime.fromtimestamp(f.stat().st_mtime)
|
||||
if mtime < cutoff:
|
||||
f.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("日志快照清理失败: %s", e)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# 容器重启事件追踪
|
||||
# ======================================================================
|
||||
|
||||
def _restart_events_file() -> Path:
|
||||
return _storage_dir() / "restart_events.jsonl"
|
||||
|
||||
|
||||
def append_restart_event(
|
||||
container: str,
|
||||
restart_before: int,
|
||||
restart_after: int,
|
||||
uptime_before: str = "",
|
||||
uptime_before_seconds: float = 0,
|
||||
trigger: str = "restart_count_increased",
|
||||
) -> None:
|
||||
"""记录一条容器重启事件到 restart_events.jsonl。"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return
|
||||
try:
|
||||
bj = timezone(timedelta(hours=8))
|
||||
entry = {
|
||||
"ts": datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"container": container,
|
||||
"restart_before": restart_before,
|
||||
"restart_after": restart_after,
|
||||
"uptime_before": uptime_before,
|
||||
"uptime_before_seconds": int(uptime_before_seconds),
|
||||
"trigger": trigger,
|
||||
}
|
||||
f = _restart_events_file()
|
||||
_append_line_locked(f, json.dumps(entry, ensure_ascii=False))
|
||||
|
||||
# 清理过期事件
|
||||
_trim_restart_events()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("重启事件写入失败: %s", e)
|
||||
|
||||
|
||||
def snapshot_on_restart(container: str, tail: int = 300) -> Optional[Path]:
|
||||
"""容器刚重启后,抓一份当前 docker logs 片段存到 logs/*-restart-*.log。"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return None
|
||||
try:
|
||||
from app.diagnostic.docker_probe import get_client
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
c = client.containers.get(container)
|
||||
raw = c.logs(tail=tail)
|
||||
text = (
|
||||
raw.decode("utf-8", errors="replace")
|
||||
if isinstance(raw, (bytes, bytearray))
|
||||
else str(raw)
|
||||
)
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
fname = f"{container}-restart-{ts}.log"
|
||||
fpath = _logs_dir() / fname
|
||||
fpath.write_text(text, encoding="utf-8")
|
||||
return fpath
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("重启日志快照失败 %s: %s", container, e)
|
||||
return None
|
||||
|
||||
|
||||
def load_restart_events(days: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
"""读取重启事件 JSONL,返回最近 N 天的行。"""
|
||||
cfg = get_diag_config()
|
||||
if not cfg.storage_enabled:
|
||||
return []
|
||||
days = days or cfg.history_retention_days
|
||||
|
||||
f = _restart_events_file()
|
||||
if not f.exists():
|
||||
return []
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
results: List[Dict[str, Any]] = []
|
||||
try:
|
||||
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
||||
for line in fp:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
ts_str = obj.get("ts", "")
|
||||
try:
|
||||
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
||||
if ts_dt >= cutoff:
|
||||
results.append(obj)
|
||||
except ValueError:
|
||||
results.append(obj)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("重启事件读取失败: %s", e)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _trim_restart_events() -> None:
|
||||
"""清理过期的重启事件行。"""
|
||||
cfg = get_diag_config()
|
||||
days = cfg.history_retention_days
|
||||
f = _restart_events_file()
|
||||
if not f.exists():
|
||||
return
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
kept: List[str] = []
|
||||
try:
|
||||
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
||||
total = 0
|
||||
for line in fp:
|
||||
total += 1
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
ts_str = obj.get("ts", "")
|
||||
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
||||
if ts_dt >= cutoff:
|
||||
kept.append(line)
|
||||
except Exception:
|
||||
kept.append(line)
|
||||
|
||||
if kept and len(kept) < total:
|
||||
_rewrite_file_locked(f, kept)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("重启事件清理失败: %s", e)
|
||||
Reference in New Issue
Block a user