feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交

核心功能:
- 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制
- FastAPI + PostgreSQL + Redis + Docker Compose 架构
- 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性
- 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页)
- 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min)

并发与稳定性修复:
- 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行)
- JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写
- 移除 SQLite fallback, 统一 PostgreSQL
- API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
2026-10-28 19:07:11 +08:00
commit 16f134e328
113 changed files with 24604 additions and 0 deletions

View File

@@ -0,0 +1,11 @@
"""MeshSafe Diagnostics Kit v1.0 — 运行时模块。"""
from app.diagnostic.config import DiagConfig, get_diag_config
from app.diagnostic.engine import DiagnosticEngine, RuleResult
__all__ = [
"DiagConfig",
"get_diag_config",
"DiagnosticEngine",
"RuleResult",
]

171
app/diagnostic/config.py Normal file
View File

@@ -0,0 +1,171 @@
"""MeshSafe Diagnostics Kit — 配置读取。
所有配置项可被环境变量覆盖(``MESHSAFE_DIAG_*`` 前缀),
优先级:环境变量 > diagnostics.yaml > 默认值。
"""
from __future__ import annotations
import functools
import os
from pathlib import Path
from typing import Any, Dict
import yaml
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_DEFAULT_YAML = _PROJECT_ROOT / "config" / "diagnostics.yaml"
def _load_yaml(path: Path) -> Dict[str, Any]:
"""读取 YAML 配置,文件不存在返回空 dict。"""
try:
with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
return data.get("diagnostics", {}) if isinstance(data, dict) else {}
except FileNotFoundError:
return {}
except Exception: # noqa: BLE001
return {}
def _env_bool(name: str, default: bool) -> bool:
v = os.environ.get(name)
if v is None:
return default
return v.strip().lower() in ("1", "true", "yes", "on")
def _env_int(name: str, default: int) -> int:
v = os.environ.get(name)
if v is None:
return default
try:
return int(v)
except ValueError:
return default
def _env_str(name: str, default: str) -> str:
return os.environ.get(name, default)
@functools.lru_cache(maxsize=1)
def get_diag_config() -> "DiagConfig":
yaml_data = _load_yaml(_DEFAULT_YAML)
return DiagConfig.from_dict(yaml_data)
class DiagConfig:
"""诊断套件配置。"""
def __init__(self, yaml_data: Dict[str, Any]):
# 顶层开关
yaml_enabled = yaml_data.get("enabled", True)
self.enabled = _env_bool(
"MESHSAFE_DIAG_ENABLED", yaml_enabled
)
# Docker 子能力
docker_yaml = yaml_data.get("docker", {})
yaml_docker_mode = docker_yaml.get("enabled", "auto")
self.docker_mode = _env_str(
"MESHSAFE_DIAG_DOCKER", yaml_docker_mode
).lower()
if self.docker_mode not in ("auto", "true", "false"):
self.docker_mode = "auto"
# 定时器周期(秒)
timers_yaml = yaml_data.get("timers", {})
self.error_scan_interval = _env_int(
"MESHSAFE_DIAG_ERROR_INTERVAL",
timers_yaml.get("error_scan_interval", 30),
)
self.container_check_interval = _env_int(
"MESHSAFE_DIAG_CONTAINER_INTERVAL",
timers_yaml.get("container_check_interval", 300),
)
self.db_aggregate_interval = _env_int(
"MESHSAFE_DIAG_DB_INTERVAL",
timers_yaml.get("db_aggregate_interval", 60),
)
# 日志行数
self.docker_log_tail = int(yaml_data.get("docker_log_tail", 500))
self.export_log_tail = int(yaml_data.get("export_log_tail", 2000))
# 导出包数据范围
export_yaml = yaml_data.get("export", {})
self.export_db_span_days = int(export_yaml.get("db_span_days", 7))
self.export_scan_tasks_max = int(export_yaml.get("scan_tasks_max", 10000))
self.export_scan_findings_max = int(export_yaml.get("scan_findings_max", 50000))
self.export_engine_errors_max = int(export_yaml.get("engine_errors_max", 5000))
# 规则阈值
rules_yaml = yaml_data.get("rules", {})
self.error_ratio_threshold = float(
rules_yaml.get("error_ratio_threshold", 0.05)
)
self.clamav_db_stale_days = int(
rules_yaml.get("clamav_db_stale_days", 7)
)
self.reject_ratio_threshold = float(
rules_yaml.get("reject_ratio_threshold", 0.20)
)
self.suspicious_exceeds_malicious = bool(
rules_yaml.get("suspicious_exceeds_malicious", True)
)
# 持久化存储
storage_yaml = yaml_data.get("storage", {})
self.storage_enabled = _env_bool(
"MESHSAFE_DIAG_STORAGE_ENABLED",
storage_yaml.get("enabled", True),
)
self.storage_dir = storage_yaml.get("dir", "/app/data/diagnostics")
self.history_retention_days = int(
storage_yaml.get("history_retention_days", 7)
)
self.log_snapshot_on_timer = bool(
storage_yaml.get("log_snapshot_on_timer", True)
)
self.log_snapshot_tail = int(
storage_yaml.get("log_snapshot_tail", 500)
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DiagConfig":
return cls(data)
def to_dict(self) -> Dict[str, Any]:
return {
"enabled": self.enabled,
"docker_mode": self.docker_mode,
"timers": {
"error_scan_interval": self.error_scan_interval,
"container_check_interval": self.container_check_interval,
"db_aggregate_interval": self.db_aggregate_interval,
},
"docker_log_tail": self.docker_log_tail,
"export_log_tail": self.export_log_tail,
"export": {
"db_span_days": self.export_db_span_days,
"scan_tasks_max": self.export_scan_tasks_max,
"scan_findings_max": self.export_scan_findings_max,
"engine_errors_max": self.export_engine_errors_max,
},
"rules": {
"error_ratio_threshold": self.error_ratio_threshold,
"clamav_db_stale_days": self.clamav_db_stale_days,
"reject_ratio_threshold": self.reject_ratio_threshold,
"suspicious_exceeds_malicious": self.suspicious_exceeds_malicious,
},
"storage": {
"enabled": self.storage_enabled,
"dir": self.storage_dir,
"history_retention_days": self.history_retention_days,
"log_snapshot_on_timer": self.log_snapshot_on_timer,
"log_snapshot_tail": self.log_snapshot_tail,
},
}

View File

@@ -0,0 +1,126 @@
"""启动时 Docker 能力检测(用 python docker SDK)。
返回三档:
full — docker.sock 可访问,日志和 inspect 都能用
partial — 能连 socket 但权限受限
none — 完全不可用(没挂 socket / 包没装)
"""
from __future__ import annotations
import os
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
_PROBE_CONTAINER = "meshsafe-api"
def _try_client() -> Optional["docker.DockerClient"]:
"""尝试创建 Docker client。失败返回 None。"""
try:
import docker # noqa: F401
except ImportError:
return None
try:
import docker
# from_env() 会自动读 DOCKER_HOST,没设的话走 unix:///var/run/docker.sock
client = docker.from_env(timeout=5)
client.ping() # 真正连一下
return client
except Exception: # noqa: BLE001
return None
def detect_docker_access(mode: str = "auto") -> Dict[str, Any]:
now = datetime.now(timezone.utc).isoformat()
sock_exists = os.path.exists("/var/run/docker.sock")
client = _try_client()
if mode == "false":
return {
"mode": "false", "status": "none",
"docker.sock_exists": sock_exists,
"docker_python_available": client is not None,
"can_logs": False, "can_inspect": False,
"containers": None,
"message": "配置强制关闭 Docker 能力(diagnostics.docker.enabled=false)",
"detected_at": now,
}
if client is None:
if mode == "true":
msg = "配置强制启用但 Docker Python SDK 不可用 — pip install docker"
else:
msg = "Docker SDK 未就绪 — 日志匹配/容器状态类规则已跳过"
return {
"mode": mode, "status": "none",
"docker.sock_exists": sock_exists,
"docker_python_available": False,
"can_logs": False, "can_inspect": False,
"containers": None,
"message": msg,
"detected_at": now,
}
# 真测
can_logs = False
can_inspect = False
containers: Optional[List[str]] = None
try:
c = client.containers.get(_PROBE_CONTAINER)
c.logs(tail=1)
can_logs = True
c.attrs # 读取属性
can_inspect = True
except Exception: # noqa: BLE001
pass
try:
containers = [c.name for c in client.containers.list(all=True)]
except Exception: # noqa: BLE001
containers = None
if can_logs and can_inspect:
status = "full"
msg = "Docker 访问正常 — 日志匹配 + 容器状态检测全部启用"
elif can_logs or can_inspect:
status = "partial"
msg = f"Docker SDK 可用但权限受限(logs={can_logs}, inspect={can_inspect})"
else:
status = "none"
msg = "docker.sock 未挂或权限不足 — 日志匹配类规则已跳过"
if mode == "true" and status != "full":
msg = f"配置强制启用但检测结果为 {status} — 检查 docker.sock 挂载"
return {
"mode": mode, "status": status,
"docker.sock_exists": sock_exists,
"docker_python_available": True,
"can_logs": can_logs,
"can_inspect": can_inspect,
"containers": containers,
"message": msg,
"detected_at": now,
}
def status_message(status: str) -> str:
return {
"full": "✅ Docker 能力完整",
"partial": "⚠️ Docker 访问受限",
"none": "ℹ️ Docker 日志检测未启用",
}.get(status, status)
def get_client():
"""获取共享 docker client(给 dumper / rules 用)。失败返回 None。"""
try:
import docker
return docker.from_env(timeout=10)
except Exception: # noqa: BLE001
return None

477
app/diagnostic/dumper.py Normal file
View File

@@ -0,0 +1,477 @@
"""一键导出诊断包(diagnostics-YYYYMMDD-HHMMSS.tar.gz)。
调用方式(HTTP 接口):
GET /api/v1/telemetry/dump → 返回文件下载
打包内容见方案文档 §3.4。
"""
from __future__ import annotations
import io
import json
import os
import subprocess
import tarfile
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from app.config import get_settings
from app.db import SessionLocal
from app.diagnostic.config import get_diag_config
from app.diagnostic.engine import DiagnosticEngine
from app.diagnostic.docker_probe import detect_docker_access
def _sanitize_filename(name: str) -> str:
"""文件名脱敏:保留扩展名 + SHA256 前 8 位。"""
if not name:
return ""
ext = os.path.splitext(name)[1].lower()
# 截取前 20 字符 + hash-like 后缀
return f"file_{hash(name) % 999999:06d}{ext}"
def _sanitize_ip(ip: str) -> str:
if not ip:
return ""
parts = ip.split(".")
if len(parts) == 4:
return f"{'.'.join(parts[:3])}.*"
return ip
def _bj_time(dt) -> Optional[str]:
"""把 DB 里的 UTC datetime(naive 或 aware)转北京时间字符串。"""
if dt is None:
return None
bj = timezone(timedelta(hours=8))
try:
# naive UTC → 加上 tz 再转
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
bj_dt = dt.astimezone(bj)
return bj_dt.strftime("%Y-%m-%d %H:%M:%S")
except Exception:
return str(dt)
def _collect_scan_tasks(days: int = 7, max_rows: int = 10000) -> List[Dict[str, Any]]:
"""拉过去 N 天的 ScanTask,脱敏后返回 dict 列表。有 max_rows 兜底。"""
from app.models import ScanTask
db = SessionLocal()
try:
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
rows = (
db.query(ScanTask)
.filter(ScanTask.created_at >= cutoff)
.order_by(ScanTask.created_at.desc())
.limit(max_rows)
.all()
)
return [
{
"id": r.id,
"filename_sanitized": _sanitize_filename(r.filename),
"declared_ext": r.declared_ext,
"size": r.size,
"sha256": r.sha256[:16],
"channel": r.channel,
"status": r.status,
"mode": r.mode,
"verdict": r.verdict,
"action": r.action,
"score": r.score,
"reason": r.reason[:500] if r.reason else "",
"policy_version": r.policy_version,
"created_at": _bj_time(r.created_at),
"finished_at": _bj_time(r.finished_at),
"duration_ms": r.duration_ms,
}
for r in rows
]
finally:
db.close()
def _collect_scan_findings(days: int = 7, max_rows: int = 50000) -> List[Dict[str, Any]]:
from app.models import ScanFinding
db = SessionLocal()
try:
# ScanFinding 没有 created_at 字段?查 ScanTask 关联时间
from app.models import ScanTask
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
rows = (
db.query(ScanFinding)
.join(ScanTask, ScanFinding.task_id == ScanTask.id)
.filter(ScanTask.created_at >= cutoff)
.order_by(ScanFinding.id.desc())
.limit(max_rows)
.all()
)
return [
{
"task_id": r.task_id,
"engine": r.engine,
"layer": r.layer,
"severity": r.severity,
"rule": r.rule,
"weight": r.weight,
"detail": (r.detail[:2000] if r.detail else ""),
"extracted_path": r.extracted_path,
}
for r in rows
]
finally:
db.close()
def _collect_engine_errors(days: int = 7, max_rows: int = 5000) -> List[Dict[str, Any]]:
from app.models import ScanTask
db = SessionLocal()
try:
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
rows = (
db.query(ScanTask)
.filter(ScanTask.verdict == "error")
.filter(ScanTask.created_at >= cutoff)
.order_by(ScanTask.created_at.desc())
.limit(max_rows)
.all()
)
return [
{
"id": r.id,
"filename_sanitized": _sanitize_filename(r.filename),
"reason": r.reason[:500] if r.reason else "",
"created_at": _bj_time(r.created_at),
}
for r in rows
]
finally:
db.close()
def _collect_policy_snapshot() -> Dict[str, Any]:
"""策略版本 + YARA 规则列表(不含规则原文)。"""
from app.main import policy_store
policy = policy_store.current()
settings = get_settings()
rules_dir = settings.rules_dir
yara_rules: List[str] = []
if os.path.isdir(rules_dir):
yara_rules = sorted(
f for f in os.listdir(rules_dir)
if f.endswith((".yar", ".yara", ".rules"))
)
return {
"version": policy.version,
"sha256": policy.sha256,
"yara_rules_dir": rules_dir,
"yara_rules_count": len(yara_rules),
"yara_rules_list": yara_rules[:50], # 最多 50 个文件名
"global": policy.raw.get("global", {}),
}
def _collect_container_logs(containers: List[str], tail: int) -> Dict[str, str]:
"""用 docker SDK 取日志。"""
from app.diagnostic.docker_probe import get_client
client = get_client()
result: Dict[str, str] = {}
for c in containers:
try:
container = client.containers.get(c)
raw = container.logs(tail=tail)
result[c] = raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
except Exception as e: # noqa: BLE001
result[c] = f"[导出失败: {e}]"
return result
def _collect_container_meta(containers: List[str]) -> Dict[str, Dict[str, Any]]:
"""用 docker SDK 取容器状态元数据(含 uptime / OOM)。"""
from app.diagnostic.docker_probe import get_client
client = get_client()
meta: Dict[str, Dict[str, Any]] = {}
now = datetime.now(timezone.utc)
for c in containers:
try:
container = client.containers.get(c)
attrs = container.attrs or {}
state = attrs.get("State", {})
started_raw = state.get("StartedAt", "")
started_at = _docker_time(started_raw)
uptime_s = (now - started_at).total_seconds() if started_at else -1
meta[c] = {
"status": state.get("Status", "unknown"),
"health": state.get("Health", {}).get("Status", ""),
"restart_count": attrs.get("RestartCount", -1),
"oom_killed": state.get("OOMKilled", False),
"exit_code": state.get("ExitCode", 0),
"started_at": _bj_time(started_at),
"uptime": _fmt_duration(uptime_s),
"uptime_seconds": int(uptime_s) if uptime_s >= 0 else -1,
"image": attrs.get("Config", {}).get("Image", ""),
}
except Exception as e: # noqa: BLE001
meta[c] = {"error": str(e)[:200]}
return meta
def _collect_compose_state() -> Dict[str, Any]:
"""用 docker SDK 列容器(替代 docker compose ps,输出结构化 JSON)。"""
from app.diagnostic.docker_probe import get_client
client = get_client()
now = datetime.now(timezone.utc)
items: List[Dict[str, Any]] = []
try:
for c in client.containers.list(all=True):
attrs = c.attrs or {}
state = attrs.get("State", {})
started_raw = state.get("StartedAt", "")
started_at = _docker_time(started_raw)
uptime_s = (now - started_at).total_seconds() if started_at else -1
items.append({
"name": c.name,
"status": state.get("Status", "unknown"),
"health": state.get("Health", {}).get("Status", ""),
"restart_count": attrs.get("RestartCount", -1),
"oom_killed": state.get("OOMKilled", False),
"exit_code": state.get("ExitCode", 0),
"image": c.image.tags[0] if c.image.tags else "unknown",
"started_at": _bj_time(started_at),
"uptime": _fmt_duration(uptime_s),
})
except Exception as e: # noqa: BLE001
return {"error": str(e)[:300], "containers": []}
return {"containers": items, "count": len(items)}
def _docker_time(raw: str) -> Optional[datetime]:
"""把 docker 返回的 ISO 时间串转 aware UTC datetime。"""
if not raw:
return None
try:
# docker 返回: "2026-10-28T05:00:12.123456789Z" 或 ".123Z"
s = raw.replace("Z", "+00:00")
# 截断纳秒到微秒(Python fromisoformat 不接受纳秒)
if "." in s:
head, frac = s.split(".", 1)
tz_idx = frac.rfind("+")
if tz_idx >= 0:
frac_ns = frac[:tz_idx]
tz = frac[tz_idx:]
frac_ns = (frac_ns + "000000")[:6]
s = head + "." + frac_ns + tz
return datetime.fromisoformat(s)
except Exception:
return None
def _fmt_duration(seconds: float) -> str:
"""把秒数格式化成人类可读的时长。"""
if seconds < 0:
return "未知"
s = int(seconds)
days, rem = divmod(s, 86400)
hours, rem = divmod(rem, 3600)
mins, secs = divmod(rem, 60)
if days > 0:
return f"{days}d {hours}h {mins}m"
if hours > 0:
return f"{hours}h {mins}m"
if mins > 0:
return f"{mins}m {secs}s"
return f"{secs}s"
# ======================================================================
# 主入口:build_diagnostics_package
# ======================================================================
def build_diagnostics_package() -> bytes:
"""生成完整诊断包,返回 tar.gz 字节。"""
started = time.time()
now = datetime.now(timezone.utc)
cfg = get_diag_config()
settings = get_settings()
containers = [
"meshsafe-api", "meshsafe-postgres",
"meshsafe-redis", "meshsafe-clamav", "meshsafe-nginx",
]
# ---- 1. docker 能力 ----
docker_info = detect_docker_access(mode=cfg.docker_mode)
# ---- 2. 导出前先存当前 docker 日志快照(持久化目录里)----
snapshot_files: List[Path] = []
try:
from app.diagnostic.storage import snapshot_container_logs
if cfg.storage_enabled and docker_info["status"] != "none":
snapshot_files = snapshot_container_logs(
containers, cfg.export_log_tail,
filename_prefix="export",
)
except Exception: # noqa: BLE001
snapshot_files = []
# ---- 3. 规则引擎(全量跑一遍)----
db = SessionLocal()
from app.main import scanner, policy_store
eng = DiagnosticEngine(
db=db, scanner=scanner, policy_store=policy_store,
docker_status=docker_info["status"],
docker_log_tail=cfg.docker_log_tail,
)
rule_results = eng.run_all(run_by="export")
db.close()
# ---- 4. 加载规则历史(持久化 JSONL)----
rule_history: List[Dict[str, Any]] = []
restart_events: List[Dict[str, Any]] = []
try:
from app.diagnostic.storage import load_rule_history, load_restart_events
if cfg.storage_enabled:
rule_history = load_rule_history(cfg.history_retention_days)
restart_events = load_restart_events(cfg.history_retention_days)
except Exception: # noqa: BLE001
rule_history = []
restart_events = []
# 按容器聚合重启事件数(用于 rules_report 的 summary)
restart_by_container: Dict[str, int] = {}
for ev in restart_events:
c = ev.get("container", "?")
restart_by_container[c] = restart_by_container.get(c, 0) + 1
# ---- 5. DB 数据(默认过去 7 天,各有上限兜底)----
span_days = getattr(cfg, "export_db_span_days", 7)
scan_tasks = _collect_scan_tasks(
days=span_days, max_rows=getattr(cfg, "export_scan_tasks_max", 10000),
)
scan_findings = _collect_scan_findings(
days=span_days, max_rows=getattr(cfg, "export_scan_findings_max", 50000),
)
engine_errors = _collect_engine_errors(
days=span_days, max_rows=getattr(cfg, "export_engine_errors_max", 5000),
)
policy_snapshot = _collect_policy_snapshot()
# ---- 6. 容器数据(如果 Docker 可用)----
docker_logs_out: Dict[str, str] = {}
container_meta: Dict[str, Dict[str, Any]] = {}
compose_state: Dict[str, Any] = {"containers": [], "error": "Docker 不可用"}
if docker_info["status"] != "none":
docker_logs_out = _collect_container_logs(containers, cfg.export_log_tail)
container_meta = _collect_container_meta(containers)
compose_state = _collect_compose_state()
# ---- 7. 组装 tar.gz ----
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
def _add_json(name: str, obj: Any) -> None:
data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8")
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
def _add_text(name: str, text: str) -> None:
data = text.encode("utf-8")
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
# meta.json
_add_json("meta.json", {
"exported_at": now.isoformat(),
"generated_by": "MeshSafe Diagnostics Kit v1.0",
"meshsafe_version": "1.1.0",
"docker_image": container_meta.get("meshsafe-api", {}).get("image", "unknown"),
"policy_version": policy_snapshot["version"],
"storage_enabled": cfg.storage_enabled,
"rules_history_count": len(rule_history),
"log_snapshots_included": len(snapshot_files),
"export_duration_ms": int((time.time() - started) * 1000),
# 数据范围
"db_span_days": getattr(cfg, "export_db_span_days", 7),
"scan_tasks_count": len(scan_tasks),
"scan_findings_count": len(scan_findings),
"engine_errors_count": len(engine_errors),
})
# docker_detect.json
_add_json("docker_detect.json", docker_info)
# compose_state.json(结构化 JSON)
_add_json("compose_state.json", compose_state)
# rules_report.json(实时全量 + 持久化历史 + 重启事件汇总)
_add_json("rules_report.json", {
"exported_at": now.isoformat(),
"docker_status": docker_info["status"],
"current_run": [r.to_dict() for r in rule_results],
"history_retention_days": cfg.history_retention_days,
"history_count": len(rule_history),
"history": rule_history,
"restart_events_count": len(restart_events),
"restart_events_summary": restart_by_container,
})
# restart_events.json(完整重启事件历史)
_add_json("restart_events.json", {
"exported_at": now.isoformat(),
"retention_days": cfg.history_retention_days,
"count": len(restart_events),
"events": restart_events,
})
# scan_tasks.json / scan_findings.json / engine_errors.json
_add_json("scan_tasks.json", scan_tasks)
_add_json("scan_findings.json", scan_findings)
_add_json("engine_errors.json", engine_errors)
# policy_snapshot.json
_add_json("policy_snapshot.json", policy_snapshot)
# engine_health.json
_add_json("engine_health.json", scanner.health())
# containers/*.log / *.meta.json(当前快照)
for c in containers:
if c in docker_logs_out:
_add_text(f"containers/{c}.log", docker_logs_out[c])
if c in container_meta:
_add_json(f"containers/{c}.meta.json", container_meta[c])
# diagnostics_logs/(持久化目录下所有 .log — timer 快照 + export 快照 + restart 快照)
try:
from app.diagnostic.storage import _logs_dir
logs_dir = _logs_dir()
if cfg.storage_enabled and logs_dir.exists():
for fpath in logs_dir.glob("*.log"):
try:
tar.add(
str(fpath),
arcname=f"diagnostics_logs/{fpath.name}",
)
except Exception: # noqa: BLE001
pass
except Exception: # noqa: BLE001
# 回退:只用 export 生成的快照
for fpath in snapshot_files:
try:
tar.add(str(fpath), arcname=f"diagnostics_logs/{fpath.name}")
except Exception:
pass
return buf.getvalue()

228
app/diagnostic/engine.py Normal file
View File

@@ -0,0 +1,228 @@
"""诊断规则引擎。
职责:
- 定义 RuleResult 数据结构(每条规则的输出格式)
- 提供 DiagnosticEngine.run_all() 入口
- 规则实现分散在 rules.py 中,engine 负责调度
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field, asdict
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
@dataclass
class RuleResult:
"""单条规则的输出。"""
id: str # R001, R002 ...
severity: str # high / medium / low / info / ok
message: str # 给用户看的中文描述
hint: str = "" # 建议/下一步操作
evidence: List[str] = field(default_factory=list) # 证据片段(日志行/DB 统计)
run_at: str = "" # ISO 格式时间戳
run_by: str = "page_request" # page_request / timer:xxx / export
skipped: bool = False # 规则是否跳过
skip_reason: Optional[str] = None # 为什么跳过(如 Docker 不可用)
def __post_init__(self) -> None:
if not self.run_at:
# 用北京时间(Asia/Shanghai, UTC+8),避免显示差 8 小时
try:
bj = timezone(timedelta(hours=8))
self.run_at = datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S")
except Exception: # noqa: BLE001
self.run_at = datetime.now(timezone.utc).isoformat()
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
# ---------------------------------------------------------------------------
# 严重性 → 徽章 CSS 类映射(和 _nav_style.html 保持一致)
# ---------------------------------------------------------------------------
SEVERITY_BADGE: Dict[str, str] = {
"high": "tag bad", # 红色
"medium": "tag warn", # 橙色
"low": "tag notice", # 黄色
"info": "tag muted", # 灰色
"ok": "tag ok", # 绿色
}
def badge_for(severity: str) -> str:
return SEVERITY_BADGE.get(severity, "tag plain")
# ---------------------------------------------------------------------------
# DiagnosticEngine — 规则调度入口
# ---------------------------------------------------------------------------
class DiagnosticEngine:
"""诊断规则引擎。
用法::
engine = DiagnosticEngine(db=SessionLocal(), scanner=scanner,
policy_store=policy_store, docker_status="full")
results = engine.run_all(run_by="page_request")
"""
def __init__(
self,
db, # SQLAlchemy Session
scanner, # Scanner 实例
policy_store, # PolicyStore 实例
docker_status: str = "none", # full / partial / none
docker_log_tail: int = 500,
):
self.db = db
self.scanner = scanner
self.policy_store = policy_store
self.docker_status = docker_status
self.docker_log_tail = docker_log_tail
# ------------------------------------------------------------------
# 入口方法
# ------------------------------------------------------------------
def run_all(self, run_by: str = "page_request") -> List[RuleResult]:
"""运行全部 15 条规则,返回结果列表。"""
# 延迟导入,避免循环依赖
from app.diagnostic import rules as _rules
results: List[RuleResult] = []
all_funcs = [
_rules.check_r001_error_ratio,
_rules.check_r002_clamav_db,
_rules.check_r003_yara_mtime,
_rules.check_r004_engine_health,
_rules.check_r005_no_scans,
_rules.check_r006_suspicious_over_malicious,
_rules.check_r007_reject_ratio,
_rules.check_r008_duplicate_sha,
# 以下 7 条需要 Docker 能力
_rules.check_r009_clamav_crash,
_rules.check_r010_postgres_fatal,
_rules.check_r011_redis_oom,
_rules.check_r012_no_space,
_rules.check_r013_nginx_upstream,
_rules.check_r014_api_connection,
_rules.check_r015_container_restart,
_rules.check_r016_restart_history,
]
for fn in all_funcs:
try:
results.append(fn(self, run_by=run_by))
except Exception as e: # noqa: BLE001
# 单条规则失败不影响整体
results.append(RuleResult(
id=getattr(fn, "_rule_id", "UNKNOWN"),
severity="info",
message=f"规则执行异常: {e}",
skipped=True,
skip_reason=str(e),
run_by=run_by,
))
return results
def run_page_request_rules(self) -> List[RuleResult]:
"""页面请求时必跑的轻量规则(R001/R002/R003/R004/R005)。"""
from app.diagnostic import rules as _rules
results: List[RuleResult] = []
for fn in [
_rules.check_r001_error_ratio,
_rules.check_r002_clamav_db,
_rules.check_r003_yara_mtime,
_rules.check_r004_engine_health,
_rules.check_r005_no_scans,
]:
try:
results.append(fn(self, run_by="page_request"))
except Exception as e: # noqa: BLE001
results.append(RuleResult(
id=getattr(fn, "_rule_id", "UNKNOWN"),
severity="info",
message=f"规则执行异常: {e}",
skipped=True,
skip_reason=str(e),
))
return results
def run_timer_error_scan(self) -> List[RuleResult]:
"""定时器 A:错误类日志(30s 周期)。"""
from app.diagnostic import rules as _rules
results: List[RuleResult] = []
for fn in [
_rules.check_r001_error_ratio,
_rules.check_r005_no_scans,
_rules.check_r009_clamav_crash,
_rules.check_r010_postgres_fatal,
_rules.check_r011_redis_oom,
_rules.check_r012_no_space,
_rules.check_r013_nginx_upstream,
_rules.check_r014_api_connection,
]:
try:
results.append(fn(self, run_by="timer:error_scan"))
except Exception as e: # noqa: BLE001
results.append(RuleResult(
id=getattr(fn, "_rule_id", "UNKNOWN"),
severity="info",
message=f"规则执行异常: {e}",
skipped=True,
skip_reason=str(e),
))
return results
def run_timer_db_aggregate(self) -> List[RuleResult]:
"""定时器 B:DB 聚合类(60s 周期)。"""
from app.diagnostic import rules as _rules
results: List[RuleResult] = []
for fn in [
_rules.check_r006_suspicious_over_malicious,
_rules.check_r007_reject_ratio,
_rules.check_r008_duplicate_sha,
]:
try:
results.append(fn(self, run_by="timer:db_aggregate"))
except Exception as e: # noqa: BLE001
results.append(RuleResult(
id=getattr(fn, "_rule_id", "UNKNOWN"),
severity="info",
message=f"规则执行异常: {e}",
skipped=True,
skip_reason=str(e),
))
return results
def run_timer_container(self) -> List[RuleResult]:
"""定时器 C:容器/引擎类(300s 周期)。"""
from app.diagnostic import rules as _rules
results: List[RuleResult] = []
for fn in [
_rules.check_r002_clamav_db,
_rules.check_r003_yara_mtime,
_rules.check_r004_engine_health,
_rules.check_r015_container_restart,
_rules.check_r016_restart_history,
]:
try:
results.append(fn(self, run_by="timer:container_check"))
except Exception as e: # noqa: BLE001
results.append(RuleResult(
id=getattr(fn, "_rule_id", "UNKNOWN"),
severity="info",
message=f"规则执行异常: {e}",
skipped=True,
skip_reason=str(e),
))
return results

878
app/diagnostic/rules.py Normal file
View File

@@ -0,0 +1,878 @@
"""诊断规则实现(15 条)。
规则按类型分四组:
页面必跑(轻量): R001 / R002 / R003 / R004 / R005
DB 聚合: R006 / R007 / R008
日志匹配: R009-R014(需要 docker.sock)
容器状态: R015(需要 docker.sock)
"""
from __future__ import annotations
import os
import subprocess
import time
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from sqlalchemy import func
from app.config import get_settings
from app.diagnostic.config import get_diag_config
from app.diagnostic.engine import DiagnosticEngine, RuleResult
# ======================================================================
# 模块级状态缓存(用于检测重启事件)
# key: container_name
# value: {"restart_count": int, "started_at": str, "checked_at": float}
# ======================================================================
_CONTAINER_LAST_STATE: Dict[str, Dict[str, Any]] = {}
def _ok(rule_id: str, message: str, hint: str = "",
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
return RuleResult(
id=rule_id, severity="ok", message=message, hint=hint,
evidence=evidence or [], run_by=run_by,
)
def _warn(rule_id: str, message: str, hint: str = "",
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
return RuleResult(
id=rule_id, severity="medium", message=message, hint=hint,
evidence=evidence or [], run_by=run_by,
)
def _crit(rule_id: str, message: str, hint: str = "",
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
return RuleResult(
id=rule_id, severity="high", message=message, hint=hint,
evidence=evidence or [], run_by=run_by,
)
def _skip(rule_id: str, reason: str, run_by: str = "") -> RuleResult:
return RuleResult(
id=rule_id, severity="info", message=f"规则未运行: {reason}",
skipped=True, skip_reason=reason, run_by=run_by,
)
# ======================================================================
# 工具函数
# ======================================================================
def _fetch_recent_verdict_counts(engine: DiagnosticEngine, limit: int = 100) -> Dict[str, int]:
"""查最近 N 次扫描的 verdict 分布。
用子查询先 limit 再 group_by,兼容 SQLAlchemy 2.0。
"""
from sqlalchemy import select
from app.models import ScanTask
# 先子查询拿到最近 N 条 verdict
sub_stmt = (
select(ScanTask.verdict)
.order_by(ScanTask.created_at.desc())
.limit(limit)
).subquery()
# 再在外层 group_by 统计
rows = engine.db.execute(
select(sub_stmt.c.verdict, func.count(sub_stmt.c.verdict))
.group_by(sub_stmt.c.verdict)
).all()
return {v or "unknown": int(c) for v, c in rows}
def _count_24h(engine: DiagnosticEngine) -> int:
from app.models import ScanTask
cutoff = datetime.now(timezone.utc) - timedelta(hours=24)
return (
engine.db.query(func.count(ScanTask.id))
.filter(ScanTask.created_at >= cutoff)
.scalar() or 0
)
def _fetch_recent_tasks(engine: DiagnosticEngine,
limit: int = 500) -> List[Any]:
from app.models import ScanTask
return (
engine.db.query(ScanTask)
.order_by(ScanTask.created_at.desc())
.limit(limit)
.all()
)
def _docker_logs(container: str, tail: int = 500) -> str:
"""用 docker SDK 取容器日志。失败返回空字符串。"""
try:
from app.diagnostic.docker_probe import get_client
client = get_client()
if not client:
return ""
c = client.containers.get(container)
raw = c.logs(tail=tail)
return raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
except Exception: # noqa: BLE001
return ""
def _docker_inspect(container: str) -> Optional[Dict[str, Any]]:
"""用 docker SDK 取容器状态元数据。失败返回 None。"""
try:
from app.diagnostic.docker_probe import get_client
client = get_client()
if not client:
return None
c = client.containers.get(container)
attrs = c.attrs or {}
state = attrs.get("State", {})
return {
"status": state.get("Status", "unknown"),
"restart_count": attrs.get("RestartCount", 0),
"started_at": state.get("StartedAt", ""),
"image": attrs.get("Config", {}).get("Image", ""),
}
except Exception: # noqa: BLE001
return None
def _grep_patterns(text: str, patterns: List[str],
max_lines: int = 20) -> List[str]:
"""从日志文本中抓取命中 pattern 的行。"""
hits: List[str] = []
for line in text.splitlines():
low = line.lower()
for p in patterns:
if p.lower() in low:
hits.append(line.strip())
break
if len(hits) >= max_lines:
break
return hits
# ======================================================================
# R001: error 比例
# ======================================================================
def check_r001_error_ratio(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
cfg = get_diag_config()
counts = _fetch_recent_verdict_counts(engine, 100)
total = sum(counts.values())
if total == 0:
return _ok("R001", "暂无扫描记录", run_by=run_by)
error_count = counts.get("error", 0)
error_ratio = error_count / total
threshold = cfg.error_ratio_threshold
if error_ratio > threshold:
return _crit(
"R001",
f"错误比例偏高({error_ratio:.1%} > 阈值 {threshold:.0%})",
f"近 100 次扫描 {error_count} 次 error",
hint="建议检查 ClamAV / Redis / Postgres 引擎健康",
evidence=[f"近 100 次 verdict 分布: {counts}"],
run_by=run_by,
)
return _ok(
"R001",
f"错误比例正常({error_ratio:.1%},阈值 {threshold:.0%})",
run_by=run_by,
)
check_r001_error_ratio._rule_id = "R001"
# ======================================================================
# R002: ClamAV 病毒库日期
# ======================================================================
def check_r002_clamav_db(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
cfg = get_diag_config()
try:
clamav_info = engine.scanner.health().get("clamav", {})
except Exception as e: # noqa: BLE001
return _skip("R002", f"ClamAV 健康检查异常: {e}", run_by=run_by)
sig_date_str = clamav_info.get("signature_date")
if not sig_date_str:
available = clamav_info.get("available", clamav_info.get("ok", False))
if not available:
return _crit(
"R002", "ClamAV 引擎不可用",
hint="检查 clamd 容器是否在运行",
run_by=run_by,
)
return _warn(
"R002", "ClamAV 病毒库日期未知",
hint="在线更新失败?U 盘离线导入",
run_by=run_by,
)
# signature_date 可能是字符串如 "2026-10-14"
try:
sig_date = datetime.strptime(str(sig_date_str)[:10], "%Y-%m-%d").date()
except Exception: # noqa: BLE001
return _skip("R002", f"无法解析病毒库日期: {sig_date_str}", run_by=run_by)
days_old = (datetime.now().date() - sig_date).days
stale = cfg.clamav_db_stale_days
if days_old > stale:
return _warn(
"R002",
f"ClamAV 病毒库已 {days_old} 天未更新",
hint="在线更新不可用 → U 盘离线导入 main.cvd / daily.cvd",
evidence=[f"signature_date = {sig_date_str}"],
run_by=run_by,
)
return _ok(
"R002",
f"ClamAV 病毒库新鲜({sig_date_str},{days_old} 天)",
run_by=run_by,
)
check_r002_clamav_db._rule_id = "R002"
# ======================================================================
# R003: YARA 规则 mtime
# ======================================================================
def check_r003_yara_mtime(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
settings = get_settings()
rules_dir = settings.rules_dir
if not os.path.isdir(rules_dir):
return _skip("R003", f"规则目录不存在: {rules_dir}", run_by=run_by)
yamls = [f for f in os.listdir(rules_dir) if f.endswith((".yar", ".yara", ".rules"))]
if not yamls:
return _warn(
"R003", "YARA 规则目录为空",
hint="检查 rules_dir 配置是否正确",
run_by=run_by,
)
latest_mtime = max(
os.path.getmtime(os.path.join(rules_dir, f)) for f in yamls
)
latest_date = datetime.fromtimestamp(latest_mtime).date()
days_old = (datetime.now().date() - latest_date).days
# 对比策略版本发布时间(如果策略文件比规则新,说明规则可能没更新)
policy_file = settings.policy_file
policy_mtime = None
if os.path.isfile(policy_file):
policy_mtime = os.path.getmtime(policy_file)
evidence = [f"规则目录: {rules_dir}", f"规则文件数: {len(yamls)}",
f"最新规则 mtime: {latest_date}"]
if policy_mtime and policy_mtime > latest_mtime:
evidence.append(f"策略文件比规则新(策略: {datetime.fromtimestamp(policy_mtime).date()})")
return _warn(
"R003",
"策略文件比 YARA 规则新,规则可能未随策略版本更新",
hint="同步更新规则文件或确认规则已加载",
evidence=evidence,
run_by=run_by,
)
return _ok(
"R003",
f"YARA 规则正常({len(yamls)} 条规则,最新 {latest_date},{days_old} 天)",
evidence=evidence,
run_by=run_by,
)
check_r003_yara_mtime._rule_id = "R003"
# ======================================================================
# R004: 引擎健康
# ======================================================================
def check_r004_engine_health(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
try:
health = engine.scanner.health()
except Exception as e: # noqa: BLE001
return _crit("R004", f"引擎健康检查异常: {e}", run_by=run_by)
offline: List[str] = []
for name, info in health.items():
ok = info.get("ok", info.get("available", False))
if not ok:
offline.append(name)
if offline:
return _crit(
"R004",
f"以下引擎不可用: {', '.join(offline)}",
hint="检查对应容器状态和网络连通",
evidence=[f"{n}: {health.get(n, {})}" for n in offline],
run_by=run_by,
)
names = list(health.keys())
return _ok(
"R004",
f"所有 {len(names)} 个引擎正常在线",
run_by=run_by,
)
check_r004_engine_health._rule_id = "R004"
# ======================================================================
# R005: 24h 无扫描
# ======================================================================
def check_r005_no_scans(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
count = _count_24h(engine)
if count == 0:
return _warn(
"R005", "过去 24 小时内没有扫描记录",
hint="摆渡系统可能停了;如果是正常停工可以忽略",
run_by=run_by,
)
return _ok(
"R005", f"过去 24 小时 {count} 次扫描",
run_by=run_by,
)
check_r005_no_scans._rule_id = "R005"
# ======================================================================
# R006: suspicious > malicious
# ======================================================================
def check_r006_suspicious_over_malicious(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
cfg = get_diag_config()
if not cfg.suspicious_exceeds_malicious:
return _skip("R006", "规则已在配置中禁用", run_by=run_by)
counts = _fetch_recent_verdict_counts(engine, 200)
suspicious = counts.get("suspicious", 0)
malicious = counts.get("malicious", 0)
if suspicious > malicious and suspicious > 0:
return _warn(
"R006",
f"可疑数({suspicious})超过恶意数({malicious})",
hint="suspicious 阈值配置可能过松;建议调紧让更多文件直接判为 malicious",
evidence=[f"近 200 次 verdict 分布: {counts}"],
run_by=run_by,
)
return _ok(
"R006",
f"suspicious={suspicious} vs malicious={malicious}",
run_by=run_by,
)
check_r006_suspicious_over_malicious._rule_id = "R006"
# ======================================================================
# R007: reject 比例
# ======================================================================
def check_r007_reject_ratio(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
cfg = get_diag_config()
counts = _fetch_recent_verdict_counts(engine, 200)
total = sum(counts.values())
if total == 0:
return _ok("R007", "暂无扫描记录", run_by=run_by)
reject = counts.get("reject", 0)
ratio = reject / total
threshold = cfg.reject_ratio_threshold
if ratio > threshold:
return _warn(
"R007",
f"策略拒绝比例偏高({ratio:.1%} > 阈值 {threshold:.0%})",
hint="检查拒绝原因 TOP;如果只是正常扩展名策略可以调宽",
evidence=[f"近 200 次 verdict 分布: {counts}"],
run_by=run_by,
)
return _ok(
"R007",
f"策略拒绝比例正常({ratio:.1%})",
run_by=run_by,
)
check_r007_reject_ratio._rule_id = "R007"
# ======================================================================
# R008: 同 SHA256 重复扫描
# ======================================================================
def check_r008_duplicate_sha(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
from app.models import ScanTask
cutoff = datetime.now(timezone.utc) - timedelta(hours=1)
rows = (
engine.db.query(ScanTask.sha256, func.count(ScanTask.id))
.filter(ScanTask.created_at >= cutoff)
.group_by(ScanTask.sha256)
.having(func.count(ScanTask.id) > 1)
.all()
)
if rows:
dups = [(sha[:12], c) for sha, c in rows]
return _crit(
"R008",
f"近 1 小时有 {len(rows)} 个 SHA256 被重复扫描",
hint="幂等机制可能失效;检查 idempotency_records 表写入",
evidence=[f"sha256={sha} count={c}" for sha, c in dups[:10]],
run_by=run_by,
)
return _ok("R008", "幂等机制正常(无重复 SHA256)", run_by=run_by)
check_r008_duplicate_sha._rule_id = "R008"
# ======================================================================
# 日志匹配型规则(需要 docker.sock)
# ======================================================================
def _check_docker_available(engine: DiagnosticEngine) -> Optional[RuleResult]:
"""如果 Docker 不可用,返回 skip RuleResult;否则返回 None(继续执行)。"""
if engine.docker_status == "none":
return _skip(None, "Docker 能力未启用", run_by="")
return None
def check_r009_clamav_crash(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R009"
return skip
cfg = get_diag_config()
logs = _docker_logs("meshsafe-clamav", cfg.docker_log_tail)
patterns = ["Killed", "exit 137", "segfault", "oom-killed", "Out of memory"]
hits = _grep_patterns(logs, patterns)
# 同时检查 docker inspect 的 restart_count
info = _docker_inspect("meshsafe-clamav")
restart_count = info.get("restart_count", 0) if info else 0
if hits or restart_count > 3:
ev = hits[:5] if hits else []
if restart_count > 3:
ev.append(f"docker inspect restart_count={restart_count}")
return _crit(
"R009",
f"ClamAV 可能崩溃(restart_count={restart_count},日志命中 {len(hits)} 行)",
hint="OOM kill → 给 clamav 容器加内存限制 512m → 1g;\nsegfault → 检查 ClamAV 版本兼容性",
evidence=ev,
run_by=run_by,
)
return _ok(
"R009",
f"ClamAV 容器稳定(restart_count={restart_count},无崩溃日志)",
run_by=run_by,
)
check_r009_clamav_crash._rule_id = "R009"
def check_r010_postgres_fatal(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R010"
return skip
cfg = get_diag_config()
logs = _docker_logs("meshsafe-postgres", cfg.docker_log_tail)
patterns = ["FATAL", "PANIC", "too many connections", "database is in recovery"]
hits = _grep_patterns(logs, patterns)
if hits:
return _crit(
"R010",
f"Postgres 出现严重错误(命中 {len(hits)} 行)",
hint="too many connections → 调大 max_connections;\nFATAL → 检查磁盘空间和 WAL",
evidence=hits[:8],
run_by=run_by,
)
return _ok("R010", "Postgres 日志无 FATAL/PANIC", run_by=run_by)
check_r010_postgres_fatal._rule_id = "R010"
def check_r011_redis_oom(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R011"
return skip
cfg = get_diag_config()
logs = _docker_logs("meshsafe-redis", cfg.docker_log_tail)
# 注意:不能用 "LOADING" — Redis 正常启动也会打 "Loading RDB / Done loading"
# 只匹配真正有问题的:加载卡死循环、OOM、panic、assertion
patterns = [
"OOM command not allowed",
"out of memory",
"LOADING loop",
"still loading",
"LOADING state",
"panic",
"Assertion",
]
hits = _grep_patterns(logs, patterns)
if hits:
return _crit(
"R011",
f"Redis 可能 OOM 或 panic(命中 {len(hits)} 行)",
hint="OOM → 调大 maxmemory 或清缓存;\nLOADING 循环 → 检查 RDB/AOF 恢复",
evidence=hits[:5],
run_by=run_by,
)
return _ok("R011", "Redis 日志无 OOM/panic", run_by=run_by)
check_r011_redis_oom._rule_id = "R011"
def check_r012_no_space(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R012"
return skip
cfg = get_diag_config()
all_patterns = ["No space left on device", "ENOSPC", "disk quota exceeded"]
containers = [
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
"meshsafe-redis", "meshsafe-nginx",
]
all_hits: List[str] = []
for c in containers:
logs = _docker_logs(c, cfg.docker_log_tail)
hits = _grep_patterns(logs, all_patterns, max_lines=5)
for h in hits:
all_hits.append(f"[{c}] {h}")
if len(all_hits) >= 10:
break
if all_hits:
return _crit(
"R012",
"容器日志出现磁盘空间不足(严重!)",
hint="立即清理磁盘,否则扫描会持续失败",
evidence=all_hits[:10],
run_by=run_by,
)
return _ok("R012", "所有容器日志无磁盘空间错误", run_by=run_by)
check_r012_no_space._rule_id = "R012"
def check_r013_nginx_upstream(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R013"
return skip
cfg = get_diag_config()
logs = _docker_logs("meshsafe-nginx", cfg.docker_log_tail)
# 只匹配真正的 upstream 故障。
# 注意 nginx 正常日志里有 "upstream response is buffered to a temporary file",
# 那是响应太大被缓冲的正常行为,绝不能在这里误伤。
patterns = [
"Connection refused",
"upstream timed out",
"upstream prematurely closed",
"no live upstreams",
"connect() failed",
"502 Bad Gateway",
"upstream server temporarily disabled",
]
hits = _grep_patterns(logs, patterns)
if hits:
return _warn(
"R013",
f"Nginx upstream 异常(命中 {len(hits)} 行)",
hint="api 容器可能临时不可用;检查 api 健康状态",
evidence=hits[:5],
run_by=run_by,
)
return _ok("R013", "Nginx 无 upstream 异常", run_by=run_by)
check_r013_nginx_upstream._rule_id = "R013"
def check_r014_api_connection(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R014"
return skip
cfg = get_diag_config()
logs = _docker_logs("meshsafe-api", cfg.docker_log_tail)
patterns = ["Redis connection refused", "Postgres connection refused",
"ConnectionError", "OperationalError", "can't connect"]
hits = _grep_patterns(logs, patterns)
if hits:
return _crit(
"R014",
f"API 与依赖服务连接异常(命中 {len(hits)} 行)",
hint="Redis/Postgres 网络断了?检查 docker 网络和依赖容器健康",
evidence=hits[:5],
run_by=run_by,
)
return _ok("R014", "API 与依赖连接正常", run_by=run_by)
check_r014_api_connection._rule_id = "R014"
# ======================================================================
# R015: 容器重启次数
# ======================================================================
def check_r015_container_restart(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
skip = _check_docker_available(engine)
if skip:
skip.id = "R015"
return skip
containers = [
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
"meshsafe-redis", "meshsafe-nginx",
]
now_ts = time.time()
bj = datetime.now(timezone(timedelta(hours=8)))
restart_events_now: List[str] = [] # 本轮检测到的新事件(写进 evidence)
new_restart_containers: List[str] = [] # 有新重启事件的容器名
for c in containers:
info = _docker_inspect(c)
if not info:
continue
cur_restart = info.get("restart_count", 0)
cur_started = info.get("started_at", "") or ""
old = _CONTAINER_LAST_STATE.get(c)
restart_changed = False
started_changed = False
if old is not None:
# 第一次跑没有 old,不算事件,只建缓存
restart_changed = cur_restart > old.get("restart_count", -1)
started_changed = cur_started != old.get("started_at", "")
if restart_changed or started_changed:
trigger = "restart_count_increased" if restart_changed else "started_at_changed"
uptime_before_s = 0
try:
uptime_before_s = int(now_ts - old.get("checked_at", now_ts))
except Exception:
uptime_before_s = 0
m, uptime_before_s = divmod(uptime_before_s, 60)
h, m = divmod(m, 60)
uptime_before_str = (
f"{h}h {m}m" if h > 0 else f"{m}m" if m > 0 else "刚启动"
)
event_line = (
f"{bj.strftime('%H:%M:%S')} {c} "
f"restart_count {old.get('restart_count','?')} → {cur_restart} "
f"uptime_before={uptime_before_str} "
f"trigger={trigger}"
)
restart_events_now.append(event_line)
new_restart_containers.append(c)
# 持久化重启事件
try:
from app.diagnostic.storage import (
append_restart_event, snapshot_on_restart,
)
append_restart_event(
container=c,
restart_before=old.get("restart_count", 0),
restart_after=cur_restart,
uptime_before=uptime_before_str,
uptime_before_seconds=now_ts - old.get("checked_at", now_ts),
trigger=trigger,
)
# 抓重启时的日志片段
try:
snapshot_on_restart(c, tail=300)
except Exception:
pass
except Exception:
pass
# 更新缓存为当前状态
_CONTAINER_LAST_STATE[c] = {
"restart_count": cur_restart,
"started_at": cur_started,
"checked_at": now_ts,
}
# ---------- 判定 severity ----------
# 1. 有新重启事件 → medium(哪怕 restart_count 还没超 3)
# 2. restart_count > 3 → crit
# 3. 否则 → ok
crit_list: List[str] = []
all_status: List[str] = []
for c in containers:
info = _docker_inspect(c)
if info:
r = info.get("restart_count", 0)
all_status.append(f"{c}: restart={r}")
if r > 3:
crit_list.append(f"{c} (restart_count={r})")
if crit_list:
# 严重:累积重启次数过多
combined_evidence = list(restart_events_now) + [f"⚠️ {x}" for x in crit_list]
return _crit(
"R015",
f"以下容器不稳定(restart_count > 3): {', '.join(crit_list)}",
hint="检查对应容器日志找 crash 根因(OOM/segmentation/配置错误)",
evidence=combined_evidence + all_status,
run_by=run_by,
)
if new_restart_containers:
# 中等:本轮检测到新重启事件(但累积次数还没超 3)
return _warn(
"R015",
f"本轮检测到容器重启事件: {', '.join(new_restart_containers)}",
hint="开发/运维可能执行了 docker restart,或容器健康检查触发了自动重启。"
"日志片段已自动归档到 diagnostics_logs/*-restart-*.log",
evidence=restart_events_now + all_status,
run_by=run_by,
)
return _ok(
"R015",
"所有容器稳定(restart_count 均 ≤ 3,本轮无新重启事件)",
evidence=all_status,
run_by=run_by,
)
check_r015_container_restart._rule_id = "R015"
# ======================================================================
# R016: 容器重启历史汇总(读持久化 restart_events.jsonl)
# ======================================================================
def check_r016_restart_history(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
"""读 restart_events.jsonl + compose_state,给出过去 N 天每个容器重启次数汇总。
和 R015 的区别:
R015 = 当前状态检测(restart_count 阈值 + 本轮变化)
R016 = 历史回顾(持久化事件日志聚合)
"""
skip = _check_docker_available(engine)
if skip:
skip.id = "R016"
return skip
cfg = get_diag_config()
# 1. 读持久化重启事件
events: List[Dict[str, Any]] = []
try:
from app.diagnostic.storage import load_restart_events
if cfg.storage_enabled:
events = load_restart_events(cfg.history_retention_days)
except Exception:
events = []
# 2. 按容器聚合
by_container: Dict[str, List[Dict[str, Any]]] = {}
for ev in events:
c = ev.get("container", "?")
by_container.setdefault(c, []).append(ev)
# 3. 取当前 restart_count(compose_state)
containers = [
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
"meshsafe-redis", "meshsafe-nginx",
]
current_counts: Dict[str, int] = {}
for c in containers:
info = _docker_inspect(c)
if info:
current_counts[c] = info.get("restart_count", 0)
# 4. 构建 evidence(每个容器一行汇总)
evidence: List[str] = []
total_events = 0
problem_containers: List[str] = []
for c in containers:
hist_count = len(by_container.get(c, []))
cur_count = current_counts.get(c, 0)
total_events += hist_count
line = f"{c}: 历史重启事件={hist_count} 当前restart_count={cur_count}"
if hist_count > 0:
last_ev = by_container[c][-1]
line += f" 最近事件={last_ev.get('ts')}"
evidence.append(line)
if hist_count >= 3:
problem_containers.append(f"{c}({hist_count}次)")
# 5. 判定
if total_events == 0:
return _ok(
"R016",
f"过去 {cfg.history_retention_days} 天内,无容器重启事件记录",
evidence=evidence,
run_by=run_by,
)
if problem_containers:
return _warn(
"R016",
f"过去 {cfg.history_retention_days} 天内,以下容器重启较频繁: "
f"{', '.join(problem_containers)}",
hint="开发/运维可能执行了多次 docker restart,"
"或容器健康检查/初始化阶段触发重启。"
"详见 restart_events.json 和 diagnostics_logs/*-restart-*.log",
evidence=evidence,
run_by=run_by,
)
# 有事件但不频繁
return _ok(
"R016",
f"过去 {cfg.history_retention_days} 天内共 {total_events} 次重启事件(均不频繁)",
evidence=evidence,
run_by=run_by,
)
check_r016_restart_history._rule_id = "R016"

268
app/diagnostic/scheduler.py Normal file
View File

@@ -0,0 +1,268 @@
"""三层定时器循环。
在 FastAPI startup 事件中用 asyncio.create_task() 启动。
定时器在后台跑,每次迭代独立 try/except + timeout,不会卡死主 event loop。
"""
from __future__ import annotations
import asyncio
import functools
import logging
import os
import subprocess
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from app.config import get_settings
from app.db import SessionLocal
from app.diagnostic.config import DiagConfig, get_diag_config
from app.diagnostic.engine import DiagnosticEngine, RuleResult
from app.services.scanner import Scanner
logger = logging.getLogger(__name__)
# 诊断套件全局运行时状态(单例 dict)
_DIAG_STATE: Dict[str, Any] = {
"enabled": False,
"docker_status": "none",
"docker_info": {},
"started_at": None,
"rules_cache": {}, # {rule_id: {result, ts, run_by}}
"latest": {"rules": [], "generated_at": None, "docker_status": "none"},
}
# 后台引用(防止 GC 收走)
_TIMER_TASKS: List[asyncio.Task] = []
# ======================================================================
# 外部访问入口
# ======================================================================
def get_diag_state() -> Dict[str, Any]:
"""获取诊断套件运行时状态(给页面/API 查询用)。"""
cache = {k: {"result": v["result"].to_dict() if isinstance(v.get("result"), RuleResult) else v.get("result"),
"ts": v.get("ts"), "run_by": v.get("run_by")}
for k, v in _DIAG_STATE["rules_cache"].items()}
return {
"enabled": _DIAG_STATE["enabled"],
"docker_status": _DIAG_STATE["docker_status"],
"docker_info": _DIAG_STATE["docker_info"],
"started_at": _DIAG_STATE["started_at"],
"rules_cache": cache,
"latest": _DIAG_STATE["latest"],
}
# ======================================================================
# 后台入口(在 app.main.startup 中调用)
# ======================================================================
def start_timers(docker_status: str, docker_info: Dict[str, Any]) -> None:
"""启动三个定时器循环(后台 daemon task)。"""
cfg = get_diag_config()
_DIAG_STATE["enabled"] = cfg.enabled
_DIAG_STATE["docker_status"] = docker_status
_DIAG_STATE["docker_info"] = docker_info
_DIAG_STATE["started_at"] = datetime.now(timezone.utc).isoformat()
if not cfg.enabled:
logger.info("MeshSafe Diagnostics: disabled")
return
loop = asyncio.get_event_loop()
if cfg.error_scan_interval > 0:
_TIMER_TASKS.append(loop.create_task(
_timer_loop("error_scan", cfg.error_scan_interval,
lambda eng: eng.run_timer_error_scan(),
docker_status, cfg.docker_log_tail),
name="diag-error-scan",
))
logger.info("诊断定时器启动: error_scan (%ss)", cfg.error_scan_interval)
if cfg.db_aggregate_interval > 0:
_TIMER_TASKS.append(loop.create_task(
_timer_loop("db_aggregate", cfg.db_aggregate_interval,
lambda eng: eng.run_timer_db_aggregate(),
docker_status, cfg.docker_log_tail),
name="diag-db-aggregate",
))
logger.info("诊断定时器启动: db_aggregate (%ss)", cfg.db_aggregate_interval)
if cfg.container_check_interval > 0:
_TIMER_TASKS.append(loop.create_task(
_timer_loop("container_check", cfg.container_check_interval,
lambda eng: eng.run_timer_container(),
docker_status, cfg.docker_log_tail),
name="diag-container-check",
))
logger.info("诊断定时器启动: container_check (%ss)", cfg.container_check_interval)
# ======================================================================
# 定时器循环(async)
# ======================================================================
async def _timer_loop(
name: str,
interval: int,
runner_fn, # 接受 DiagnosticEngine 返回 List[RuleResult]
docker_status: str,
docker_log_tail: int,
) -> None:
"""通用定时器循环。每次迭代独立 try/except + 超时保护。"""
# 等一个 interval 再开始第一次(避免 startup 时 DB 还没 ready)
await asyncio.sleep(min(interval, 15))
while True:
try:
# 分布式锁(时间槽位):多 worker 部署时,每个时间槽只有
# 一个 worker 能拿到锁并执行,保证执行频率精确等于 interval。
if not _acquire_timer_lock(name, interval):
logger.debug("诊断定时器 %s 本轮跳过(其他 worker 持锁)", name)
await asyncio.sleep(interval)
continue
results = await asyncio.wait_for(
asyncio.get_event_loop().run_in_executor(
None,
lambda: _run_sync(name, runner_fn, docker_status, docker_log_tail),
),
timeout=30.0,
)
_cache_results(name, results, docker_status)
logger.debug("诊断定时器 %s 跑完: %d 条规则", name, len(results))
except asyncio.CancelledError:
logger.info("诊断定时器 %s 被取消", name)
break
except asyncio.TimeoutError:
logger.warning("诊断定时器 %s 超时(30s),本轮跳过", name)
except Exception as e: # noqa: BLE001
logger.exception("诊断定时器 %s 异常: %s,%ss 后重试", name, e, interval)
await asyncio.sleep(interval)
# ======================================================================
# 定时器分布式锁(Redis)
#
# 多 worker 部署(UVICORN_WORKERS > 1)时,每个 worker 都会启动定时器,
# 导致:同一轮规则被跑 N 次、历史数据量 ×N、并发写 JSONL 造成文件损坏。
# 用 Redis SET NX 抢锁,保证同一时刻只有一个 worker 在跑。
# ======================================================================
_TIMER_LOCK_PREFIX = "meshsafe:diag:timer_lock:"
@functools.lru_cache(maxsize=1)
def _lock_redis():
"""定时器锁专用 Redis 客户端(进程内复用,避免每次调用重建连接)。"""
import redis # type: ignore
from app.config import get_settings
return redis.from_url(
get_settings().redis_url, decode_responses=True, socket_timeout=3
)
def _acquire_timer_lock(timer_name: str, interval: int) -> bool:
"""尝试获取定时器锁(时间槽位设计)。
key 里带上当前时间槽位号:``meshsafe:diag:timer_lock:<timer>:<slot>``,
slot = 当前时间 // interval。这样无论多少 worker、谁先谁后,
同一个槽位内最多只有一个 worker 能拿到锁并执行,
下一个槽位自然放行 —— 执行频率精确等于 interval,不多不少。
Redis 不可用时返回 True(降级为单 worker 行为,不阻塞主流程)。
"""
try:
client = _lock_redis()
slot = int(time.time() // interval)
key = f"{_TIMER_LOCK_PREFIX}{timer_name}:{slot}"
# EX 只做兜底(槽位 key 不会互相干扰),设 2 倍 interval 足够
return bool(client.set(key, "1", nx=True, ex=interval * 2))
except Exception as e: # noqa: BLE001
# Redis 不可用 → 不加锁(单机单 worker 场景行为不变)
logger.debug("定时器锁获取失败,降级为无锁执行: %s", e)
return True
def _run_sync(
name: str,
runner_fn,
docker_status: str,
docker_log_tail: int,
) -> List[RuleResult]:
"""在线程池里同步跑规则(避免阻塞 event loop)。"""
db = SessionLocal()
try:
# 延迟 import 避免循环依赖
from app.main import scanner, policy_store
eng = DiagnosticEngine(
db=db, scanner=scanner, policy_store=policy_store,
docker_status=docker_status, docker_log_tail=docker_log_tail,
)
results = runner_fn(eng)
return results
finally:
db.close()
def _cache_results(
timer_name: str,
results: List[RuleResult],
docker_status: str,
) -> None:
"""把本轮规则结果写入内存缓存(dict)+ 持久化存储。"""
now = time.time()
for r in results:
_DIAG_STATE["rules_cache"][r.id] = {
"result": r,
"ts": now,
"run_by": r.run_by or f"timer:{timer_name}",
}
# 构建 latest 快照(合并所有缓存 + 本轮结果)
all_results = []
seen = set()
for rid, entry in _DIAG_STATE["rules_cache"].items():
if rid not in seen and entry.get("result"):
all_results.append(entry["result"])
seen.add(rid)
_DIAG_STATE["latest"] = {
"rules": sorted(all_results, key=lambda x: x.id),
"generated_at": now,
"docker_status": docker_status,
}
# ---------- 持久化 ----------
try:
from app.diagnostic.config import get_diag_config
from app.diagnostic.storage import append_rule_results
cfg = get_diag_config()
if cfg.storage_enabled:
append_rule_results(results, timer_name)
# 可选:定时器也存 docker 日志快照(省磁盘所以用较小 tail)
if cfg.log_snapshot_on_timer and docker_status != "none":
from app.diagnostic.storage import snapshot_container_logs
containers = [
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
"meshsafe-redis", "meshsafe-nginx",
]
# 不要每次都存 — 只在 container_check(5 分钟周期)里存一次
if timer_name == "container_check":
snapshot_container_logs(
containers, cfg.log_snapshot_tail,
filename_prefix="timer",
)
except Exception as e: # noqa: BLE001
logger.debug("诊断持久化跳过(非致命): %s", e)

455
app/diagnostic/storage.py Normal file
View File

@@ -0,0 +1,455 @@
"""诊断套件持久化存储模块。
职责:
- 定时器规则结果 → JSON Lines 追加(rules_history.jsonl)
- Docker 日志快照 → 存文件(logs/*.log)
- 自动清理过期数据(基于配置的 history_retention_days)
"""
from __future__ import annotations
import json
import logging
import os
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from app.diagnostic.config import get_diag_config
logger = logging.getLogger(__name__)
def _storage_dir() -> Path:
"""获取存储根目录。"""
cfg = get_diag_config()
p = Path(cfg.storage_dir)
p.mkdir(parents=True, exist_ok=True)
return p
def _logs_dir() -> Path:
p = _storage_dir() / "logs"
p.mkdir(parents=True, exist_ok=True)
return p
def _rules_history_file() -> Path:
return _storage_dir() / "rules_history.jsonl"
# ======================================================================
# 跨进程文件锁写入
#
# 多 worker 部署时,每个 worker 都会跑诊断定时器。"a" 追加模式在多进程
# 下不安全——各进程独立维护文件偏移量,后写的会覆盖先写的内容,把一行
# JSON 从中间劈开,产出非法 UTF-8 字节,导致后续读取全部报错。
# 这里用 fcntl 文件锁保证同一时刻只有一个进程在写。
# ======================================================================
try:
import fcntl # type: ignore
_HAS_FCNTL = True
except ImportError: # pragma: no cover - Windows 平台
fcntl = None # type: ignore
_HAS_FCNTL = False
def _append_line_locked(path: Path, line: str) -> None:
"""向文件追加一行,跨进程互斥。
Linux 下用 fcntl.flock 排他锁;无 fcntl 时退化为进程内线程锁。
"""
payload = line + "\n"
with open(path, "a", encoding="utf-8") as fp:
if _HAS_FCNTL:
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
try:
fp.seek(0, os.SEEK_END)
fp.write(payload)
fp.flush()
os.fsync(fp.fileno())
finally:
if _HAS_FCNTL:
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
def _rewrite_file_locked(path: Path, lines: List[str]) -> None:
"""整文件重写(清理过期数据用),同样走文件锁。
写临时文件再原子替换,避免重写过程中其他进程读到半截内容。
"""
tmp = path.with_suffix(path.suffix + ".tmp")
with open(tmp, "w", encoding="utf-8") as fp:
if _HAS_FCNTL:
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
try:
for line in lines:
fp.write(line + "\n")
fp.flush()
os.fsync(fp.fileno())
finally:
if _HAS_FCNTL:
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
os.replace(tmp, path)
# ======================================================================
# 规则历史:JSON Lines 追加
# ======================================================================
def append_rule_results(
rules: List[Any],
timer_name: str,
) -> None:
"""定时器每次跑完追加一行 JSONL。
每行格式::
{"ts": "2026-10-28T13:12:14", "timer": "error_scan",
"docker_status": "full",
"rules": [{"id":"R001","severity":"ok","message":"...","run_at":"...","run_by":"timer:error_scan",...}]}
"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return
try:
# 北京时间
bj = timezone(timedelta(hours=8))
entry: Dict[str, Any] = {
"ts": datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S"),
"timer": timer_name,
"docker_status": None,
"rules": [
r.to_dict() if hasattr(r, "to_dict") else dict(r)
for r in rules
],
}
f = _rules_history_file()
_append_line_locked(f, json.dumps(entry, ensure_ascii=False))
# 顺带清理过期
_trim_history()
except Exception as e: # noqa: BLE001
logger.warning("诊断持久化写入失败: %s", e)
def load_rule_history(days: Optional[int] = None) -> List[Dict[str, Any]]:
"""读取规则历史 JSONL,返回最近 N 天的行。"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return []
days = days or cfg.history_retention_days
f = _rules_history_file()
if not f.exists():
return []
cutoff = datetime.now() - timedelta(days=days)
results: List[Dict[str, Any]] = []
try:
with open(f, "r", encoding="utf-8", errors="replace") as fp:
for line in fp:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
# 按 ts 判断是否过期
ts_str = obj.get("ts", "")
try:
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
if ts_dt >= cutoff:
results.append(obj)
except ValueError:
# 时间格式不对就不过滤,保留
results.append(obj)
except Exception as e: # noqa: BLE001
logger.warning("诊断历史读取失败: %s", e)
return results
def _trim_history() -> None:
"""清理过期的规则历史行(保留文件头)。"""
cfg = get_diag_config()
days = cfg.history_retention_days
f = _rules_history_file()
if not f.exists():
return
cutoff = datetime.now() - timedelta(days=days)
kept: List[str] = []
try:
with open(f, "r", encoding="utf-8", errors="replace") as fp:
total = 0
for line in fp:
total += 1
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
ts_str = obj.get("ts", "")
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
if ts_dt >= cutoff:
kept.append(line)
except Exception:
kept.append(line) # 格式异常的保留
# 如果删了一些(不是全部),就重写
if kept and len(kept) < total:
_rewrite_file_locked(f, kept)
except Exception as e: # noqa: BLE001
logger.warning("诊断历史清理失败: %s", e)
# ======================================================================
# Docker 日志快照
# ======================================================================
def snapshot_container_logs(
containers: List[str],
tail: int,
filename_prefix: Optional[str] = None,
) -> List[Path]:
"""把各容器当前 docker logs --tail N 存一份文件。
文件名: {container}-{prefix}{timestamp}.log
prefix 可选(如 timer 存的加前缀 timer-)
返回写入的文件列表。
"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return []
from app.diagnostic.docker_probe import get_client
client = get_client()
if client is None:
return []
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
prefix = f"{filename_prefix}-" if filename_prefix else ""
out: List[Path] = []
for name in containers:
try:
c = client.containers.get(name)
raw = c.logs(tail=tail)
text = (
raw.decode("utf-8", errors="replace")
if isinstance(raw, (bytes, bytearray))
else str(raw)
)
fname = f"{name}-{prefix}{ts}.log"
fpath = _logs_dir() / fname
fpath.write_text(text, encoding="utf-8")
out.append(fpath)
except Exception as e: # noqa: BLE001
logger.warning("容器 %s 日志快照失败: %s", name, e)
# 顺带清理过期日志
_trim_logs()
return out
def load_log_snapshots(containers: Optional[List[str]] = None) -> Dict[str, List[Path]]:
"""读 logs/ 目录,返回 {container_name: [path, path, ...]}。"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return {}
cutoff = datetime.now() - timedelta(days=cfg.history_retention_days)
result: Dict[str, List[Path]] = {}
for f in _logs_dir().glob("*.log"):
stem = f.stem # 去掉 .log
# 文件名格式: meshsafe-api-timer-20261028-1312 或 meshsafe-api-20261028-1312
# 容器名是 stem 里第一个 - 前面的部分... 其实容器名本身也可能带 -
# 简化:直接用 stem 整体当 key(导出时按需匹配)
# 但我们需要按容器分组,所以用 glob 前缀匹配
if containers:
matched = None
for c in containers:
if stem.startswith(c + "-"):
matched = c
break
if matched is None:
continue
container_key = matched
else:
# 没指定容器,尝试从文件名推断
parts = stem.split("-")
container_key = parts[0] if parts else stem
try:
mtime = datetime.fromtimestamp(f.stat().st_mtime)
if mtime >= cutoff:
result.setdefault(container_key, []).append(f)
except Exception:
result.setdefault(container_key, []).append(f)
# 按 mtime 排序(新的在前)
for k in result:
result[k].sort(key=lambda p: p.stat().st_mtime, reverse=True)
return result
def _trim_logs() -> None:
"""清理过期的日志快照文件。"""
cfg = get_diag_config()
cutoff = datetime.now() - timedelta(days=cfg.history_retention_days)
try:
for f in _logs_dir().glob("*.log"):
try:
mtime = datetime.fromtimestamp(f.stat().st_mtime)
if mtime < cutoff:
f.unlink()
except FileNotFoundError:
pass
except Exception as e: # noqa: BLE001
logger.warning("日志快照清理失败: %s", e)
# ======================================================================
# 容器重启事件追踪
# ======================================================================
def _restart_events_file() -> Path:
return _storage_dir() / "restart_events.jsonl"
def append_restart_event(
container: str,
restart_before: int,
restart_after: int,
uptime_before: str = "",
uptime_before_seconds: float = 0,
trigger: str = "restart_count_increased",
) -> None:
"""记录一条容器重启事件到 restart_events.jsonl。"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return
try:
bj = timezone(timedelta(hours=8))
entry = {
"ts": datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S"),
"container": container,
"restart_before": restart_before,
"restart_after": restart_after,
"uptime_before": uptime_before,
"uptime_before_seconds": int(uptime_before_seconds),
"trigger": trigger,
}
f = _restart_events_file()
_append_line_locked(f, json.dumps(entry, ensure_ascii=False))
# 清理过期事件
_trim_restart_events()
except Exception as e: # noqa: BLE001
logger.warning("重启事件写入失败: %s", e)
def snapshot_on_restart(container: str, tail: int = 300) -> Optional[Path]:
"""容器刚重启后,抓一份当前 docker logs 片段存到 logs/*-restart-*.log。"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return None
try:
from app.diagnostic.docker_probe import get_client
client = get_client()
if client is None:
return None
c = client.containers.get(container)
raw = c.logs(tail=tail)
text = (
raw.decode("utf-8", errors="replace")
if isinstance(raw, (bytes, bytearray))
else str(raw)
)
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
fname = f"{container}-restart-{ts}.log"
fpath = _logs_dir() / fname
fpath.write_text(text, encoding="utf-8")
return fpath
except Exception as e: # noqa: BLE001
logger.warning("重启日志快照失败 %s: %s", container, e)
return None
def load_restart_events(days: Optional[int] = None) -> List[Dict[str, Any]]:
"""读取重启事件 JSONL,返回最近 N 天的行。"""
cfg = get_diag_config()
if not cfg.storage_enabled:
return []
days = days or cfg.history_retention_days
f = _restart_events_file()
if not f.exists():
return []
cutoff = datetime.now() - timedelta(days=days)
results: List[Dict[str, Any]] = []
try:
with open(f, "r", encoding="utf-8", errors="replace") as fp:
for line in fp:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
ts_str = obj.get("ts", "")
try:
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
if ts_dt >= cutoff:
results.append(obj)
except ValueError:
results.append(obj)
except Exception as e: # noqa: BLE001
logger.warning("重启事件读取失败: %s", e)
return results
def _trim_restart_events() -> None:
"""清理过期的重启事件行。"""
cfg = get_diag_config()
days = cfg.history_retention_days
f = _restart_events_file()
if not f.exists():
return
cutoff = datetime.now() - timedelta(days=days)
kept: List[str] = []
try:
with open(f, "r", encoding="utf-8", errors="replace") as fp:
total = 0
for line in fp:
total += 1
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
ts_str = obj.get("ts", "")
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
if ts_dt >= cutoff:
kept.append(line)
except Exception:
kept.append(line)
if kept and len(kept) < total:
_rewrite_file_locked(f, kept)
except Exception as e: # noqa: BLE001
logger.warning("重启事件清理失败: %s", e)