feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交

核心功能: - 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制 - FastAPI + PostgreSQL + Redis + Docker Compose 架构 - 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性 - 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页) - 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min) 并发与稳定性修复: - 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行) - JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写 - 移除 SQLite fallback, 统一 PostgreSQL - API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
2026-10-28 19:07:11 +08:00
commit 16f134e328
113 changed files with 24604 additions and 0 deletions
--- a/app/diagnostic/dumper.py
+++ b/app/diagnostic/dumper.py
@@ -0,0 +1,477 @@
+"""一键导出诊断包（diagnostics-YYYYMMDD-HHMMSS.tar.gz）。
+
+调用方式（HTTP 接口）:
+    GET /api/v1/telemetry/dump  → 返回文件下载
+
+打包内容见方案文档 §3.4。
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import os
+import subprocess
+import tarfile
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from app.config import get_settings
+from app.db import SessionLocal
+from app.diagnostic.config import get_diag_config
+from app.diagnostic.engine import DiagnosticEngine
+from app.diagnostic.docker_probe import detect_docker_access
+
+
+def _sanitize_filename(name: str) -> str:
+    """文件名脱敏：保留扩展名 + SHA256 前 8 位。"""
+    if not name:
+        return ""
+    ext = os.path.splitext(name)[1].lower()
+    # 截取前 20 字符 + hash-like 后缀
+    return f"file_{hash(name) % 999999:06d}{ext}"
+
+
+def _sanitize_ip(ip: str) -> str:
+    if not ip:
+        return ""
+    parts = ip.split(".")
+    if len(parts) == 4:
+        return f"{'.'.join(parts[:3])}.*"
+    return ip
+
+
+def _bj_time(dt) -> Optional[str]:
+    """把 DB 里的 UTC datetime（naive 或 aware）转北京时间字符串。"""
+    if dt is None:
+        return None
+    bj = timezone(timedelta(hours=8))
+    try:
+        # naive UTC → 加上 tz 再转
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        bj_dt = dt.astimezone(bj)
+        return bj_dt.strftime("%Y-%m-%d %H:%M:%S")
+    except Exception:
+        return str(dt)
+
+
+def _collect_scan_tasks(days: int = 7, max_rows: int = 10000) -> List[Dict[str, Any]]:
+    """拉过去 N 天的 ScanTask，脱敏后返回 dict 列表。有 max_rows 兜底。"""
+    from app.models import ScanTask
+    db = SessionLocal()
+    try:
+        cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+        rows = (
+            db.query(ScanTask)
+            .filter(ScanTask.created_at >= cutoff)
+            .order_by(ScanTask.created_at.desc())
+            .limit(max_rows)
+            .all()
+        )
+        return [
+            {
+                "id": r.id,
+                "filename_sanitized": _sanitize_filename(r.filename),
+                "declared_ext": r.declared_ext,
+                "size": r.size,
+                "sha256": r.sha256[:16],
+                "channel": r.channel,
+                "status": r.status,
+                "mode": r.mode,
+                "verdict": r.verdict,
+                "action": r.action,
+                "score": r.score,
+                "reason": r.reason[:500] if r.reason else "",
+                "policy_version": r.policy_version,
+                "created_at": _bj_time(r.created_at),
+                "finished_at": _bj_time(r.finished_at),
+                "duration_ms": r.duration_ms,
+            }
+            for r in rows
+        ]
+    finally:
+        db.close()
+
+
+def _collect_scan_findings(days: int = 7, max_rows: int = 50000) -> List[Dict[str, Any]]:
+    from app.models import ScanFinding
+    db = SessionLocal()
+    try:
+        # ScanFinding 没有 created_at 字段？查 ScanTask 关联时间
+        from app.models import ScanTask
+        cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+        rows = (
+            db.query(ScanFinding)
+            .join(ScanTask, ScanFinding.task_id == ScanTask.id)
+            .filter(ScanTask.created_at >= cutoff)
+            .order_by(ScanFinding.id.desc())
+            .limit(max_rows)
+            .all()
+        )
+        return [
+            {
+                "task_id": r.task_id,
+                "engine": r.engine,
+                "layer": r.layer,
+                "severity": r.severity,
+                "rule": r.rule,
+                "weight": r.weight,
+                "detail": (r.detail[:2000] if r.detail else ""),
+                "extracted_path": r.extracted_path,
+            }
+            for r in rows
+        ]
+    finally:
+        db.close()
+
+
+def _collect_engine_errors(days: int = 7, max_rows: int = 5000) -> List[Dict[str, Any]]:
+    from app.models import ScanTask
+    db = SessionLocal()
+    try:
+        cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+        rows = (
+            db.query(ScanTask)
+            .filter(ScanTask.verdict == "error")
+            .filter(ScanTask.created_at >= cutoff)
+            .order_by(ScanTask.created_at.desc())
+            .limit(max_rows)
+            .all()
+        )
+        return [
+            {
+                "id": r.id,
+                "filename_sanitized": _sanitize_filename(r.filename),
+                "reason": r.reason[:500] if r.reason else "",
+                "created_at": _bj_time(r.created_at),
+            }
+            for r in rows
+        ]
+    finally:
+        db.close()
+
+
+def _collect_policy_snapshot() -> Dict[str, Any]:
+    """策略版本 + YARA 规则列表（不含规则原文）。"""
+    from app.main import policy_store
+    policy = policy_store.current()
+    settings = get_settings()
+    rules_dir = settings.rules_dir
+
+    yara_rules: List[str] = []
+    if os.path.isdir(rules_dir):
+        yara_rules = sorted(
+            f for f in os.listdir(rules_dir)
+            if f.endswith((".yar", ".yara", ".rules"))
+        )
+
+    return {
+        "version": policy.version,
+        "sha256": policy.sha256,
+        "yara_rules_dir": rules_dir,
+        "yara_rules_count": len(yara_rules),
+        "yara_rules_list": yara_rules[:50],  # 最多 50 个文件名
+        "global": policy.raw.get("global", {}),
+    }
+
+
+def _collect_container_logs(containers: List[str], tail: int) -> Dict[str, str]:
+    """用 docker SDK 取日志。"""
+    from app.diagnostic.docker_probe import get_client
+    client = get_client()
+    result: Dict[str, str] = {}
+    for c in containers:
+        try:
+            container = client.containers.get(c)
+            raw = container.logs(tail=tail)
+            result[c] = raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
+        except Exception as e:  # noqa: BLE001
+            result[c] = f"[导出失败: {e}]"
+    return result
+
+
+def _collect_container_meta(containers: List[str]) -> Dict[str, Dict[str, Any]]:
+    """用 docker SDK 取容器状态元数据（含 uptime / OOM）。"""
+    from app.diagnostic.docker_probe import get_client
+    client = get_client()
+    meta: Dict[str, Dict[str, Any]] = {}
+    now = datetime.now(timezone.utc)
+    for c in containers:
+        try:
+            container = client.containers.get(c)
+            attrs = container.attrs or {}
+            state = attrs.get("State", {})
+            started_raw = state.get("StartedAt", "")
+            started_at = _docker_time(started_raw)
+            uptime_s = (now - started_at).total_seconds() if started_at else -1
+
+            meta[c] = {
+                "status": state.get("Status", "unknown"),
+                "health": state.get("Health", {}).get("Status", ""),
+                "restart_count": attrs.get("RestartCount", -1),
+                "oom_killed": state.get("OOMKilled", False),
+                "exit_code": state.get("ExitCode", 0),
+                "started_at": _bj_time(started_at),
+                "uptime": _fmt_duration(uptime_s),
+                "uptime_seconds": int(uptime_s) if uptime_s >= 0 else -1,
+                "image": attrs.get("Config", {}).get("Image", ""),
+            }
+        except Exception as e:  # noqa: BLE001
+            meta[c] = {"error": str(e)[:200]}
+    return meta
+
+
+def _collect_compose_state() -> Dict[str, Any]:
+    """用 docker SDK 列容器（替代 docker compose ps，输出结构化 JSON）。"""
+    from app.diagnostic.docker_probe import get_client
+    client = get_client()
+    now = datetime.now(timezone.utc)
+    items: List[Dict[str, Any]] = []
+    try:
+        for c in client.containers.list(all=True):
+            attrs = c.attrs or {}
+            state = attrs.get("State", {})
+            started_raw = state.get("StartedAt", "")
+            started_at = _docker_time(started_raw)
+            uptime_s = (now - started_at).total_seconds() if started_at else -1
+
+            items.append({
+                "name": c.name,
+                "status": state.get("Status", "unknown"),
+                "health": state.get("Health", {}).get("Status", ""),
+                "restart_count": attrs.get("RestartCount", -1),
+                "oom_killed": state.get("OOMKilled", False),
+                "exit_code": state.get("ExitCode", 0),
+                "image": c.image.tags[0] if c.image.tags else "unknown",
+                "started_at": _bj_time(started_at),
+                "uptime": _fmt_duration(uptime_s),
+            })
+    except Exception as e:  # noqa: BLE001
+        return {"error": str(e)[:300], "containers": []}
+    return {"containers": items, "count": len(items)}
+
+
+def _docker_time(raw: str) -> Optional[datetime]:
+    """把 docker 返回的 ISO 时间串转 aware UTC datetime。"""
+    if not raw:
+        return None
+    try:
+        # docker 返回: "2026-10-28T05:00:12.123456789Z" 或 ".123Z"
+        s = raw.replace("Z", "+00:00")
+        # 截断纳秒到微秒（Python fromisoformat 不接受纳秒）
+        if "." in s:
+            head, frac = s.split(".", 1)
+            tz_idx = frac.rfind("+")
+            if tz_idx >= 0:
+                frac_ns = frac[:tz_idx]
+                tz = frac[tz_idx:]
+                frac_ns = (frac_ns + "000000")[:6]
+                s = head + "." + frac_ns + tz
+        return datetime.fromisoformat(s)
+    except Exception:
+        return None
+
+
+def _fmt_duration(seconds: float) -> str:
+    """把秒数格式化成人类可读的时长。"""
+    if seconds < 0:
+        return "未知"
+    s = int(seconds)
+    days, rem = divmod(s, 86400)
+    hours, rem = divmod(rem, 3600)
+    mins, secs = divmod(rem, 60)
+    if days > 0:
+        return f"{days}d {hours}h {mins}m"
+    if hours > 0:
+        return f"{hours}h {mins}m"
+    if mins > 0:
+        return f"{mins}m {secs}s"
+    return f"{secs}s"
+
+
+# ======================================================================
+# 主入口：build_diagnostics_package
+# ======================================================================
+
+def build_diagnostics_package() -> bytes:
+    """生成完整诊断包，返回 tar.gz 字节。"""
+
+    started = time.time()
+    now = datetime.now(timezone.utc)
+    cfg = get_diag_config()
+    settings = get_settings()
+
+    containers = [
+        "meshsafe-api", "meshsafe-postgres",
+        "meshsafe-redis", "meshsafe-clamav", "meshsafe-nginx",
+    ]
+
+    # ---- 1. docker 能力 ----
+    docker_info = detect_docker_access(mode=cfg.docker_mode)
+
+    # ---- 2. 导出前先存当前 docker 日志快照（持久化目录里）----
+    snapshot_files: List[Path] = []
+    try:
+        from app.diagnostic.storage import snapshot_container_logs
+        if cfg.storage_enabled and docker_info["status"] != "none":
+            snapshot_files = snapshot_container_logs(
+                containers, cfg.export_log_tail,
+                filename_prefix="export",
+            )
+    except Exception:  # noqa: BLE001
+        snapshot_files = []
+
+    # ---- 3. 规则引擎（全量跑一遍）----
+    db = SessionLocal()
+    from app.main import scanner, policy_store
+    eng = DiagnosticEngine(
+        db=db, scanner=scanner, policy_store=policy_store,
+        docker_status=docker_info["status"],
+        docker_log_tail=cfg.docker_log_tail,
+    )
+    rule_results = eng.run_all(run_by="export")
+    db.close()
+
+    # ---- 4. 加载规则历史（持久化 JSONL）----
+    rule_history: List[Dict[str, Any]] = []
+    restart_events: List[Dict[str, Any]] = []
+    try:
+        from app.diagnostic.storage import load_rule_history, load_restart_events
+        if cfg.storage_enabled:
+            rule_history = load_rule_history(cfg.history_retention_days)
+            restart_events = load_restart_events(cfg.history_retention_days)
+    except Exception:  # noqa: BLE001
+        rule_history = []
+        restart_events = []
+
+    # 按容器聚合重启事件数（用于 rules_report 的 summary）
+    restart_by_container: Dict[str, int] = {}
+    for ev in restart_events:
+        c = ev.get("container", "?")
+        restart_by_container[c] = restart_by_container.get(c, 0) + 1
+
+    # ---- 5. DB 数据（默认过去 7 天，各有上限兜底）----
+    span_days = getattr(cfg, "export_db_span_days", 7)
+    scan_tasks = _collect_scan_tasks(
+        days=span_days, max_rows=getattr(cfg, "export_scan_tasks_max", 10000),
+    )
+    scan_findings = _collect_scan_findings(
+        days=span_days, max_rows=getattr(cfg, "export_scan_findings_max", 50000),
+    )
+    engine_errors = _collect_engine_errors(
+        days=span_days, max_rows=getattr(cfg, "export_engine_errors_max", 5000),
+    )
+    policy_snapshot = _collect_policy_snapshot()
+
+    # ---- 6. 容器数据（如果 Docker 可用）----
+    docker_logs_out: Dict[str, str] = {}
+    container_meta: Dict[str, Dict[str, Any]] = {}
+    compose_state: Dict[str, Any] = {"containers": [], "error": "Docker 不可用"}
+    if docker_info["status"] != "none":
+        docker_logs_out = _collect_container_logs(containers, cfg.export_log_tail)
+        container_meta = _collect_container_meta(containers)
+        compose_state = _collect_compose_state()
+
+    # ---- 7. 组装 tar.gz ----
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+
+        def _add_json(name: str, obj: Any) -> None:
+            data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8")
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            tar.addfile(info, io.BytesIO(data))
+
+        def _add_text(name: str, text: str) -> None:
+            data = text.encode("utf-8")
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            tar.addfile(info, io.BytesIO(data))
+
+        # meta.json
+        _add_json("meta.json", {
+            "exported_at": now.isoformat(),
+            "generated_by": "MeshSafe Diagnostics Kit v1.0",
+            "meshsafe_version": "1.1.0",
+            "docker_image": container_meta.get("meshsafe-api", {}).get("image", "unknown"),
+            "policy_version": policy_snapshot["version"],
+            "storage_enabled": cfg.storage_enabled,
+            "rules_history_count": len(rule_history),
+            "log_snapshots_included": len(snapshot_files),
+            "export_duration_ms": int((time.time() - started) * 1000),
+            # 数据范围
+            "db_span_days": getattr(cfg, "export_db_span_days", 7),
+            "scan_tasks_count": len(scan_tasks),
+            "scan_findings_count": len(scan_findings),
+            "engine_errors_count": len(engine_errors),
+        })
+
+        # docker_detect.json
+        _add_json("docker_detect.json", docker_info)
+
+        # compose_state.json（结构化 JSON）
+        _add_json("compose_state.json", compose_state)
+
+        # rules_report.json（实时全量 + 持久化历史 + 重启事件汇总）
+        _add_json("rules_report.json", {
+            "exported_at": now.isoformat(),
+            "docker_status": docker_info["status"],
+            "current_run": [r.to_dict() for r in rule_results],
+            "history_retention_days": cfg.history_retention_days,
+            "history_count": len(rule_history),
+            "history": rule_history,
+            "restart_events_count": len(restart_events),
+            "restart_events_summary": restart_by_container,
+        })
+
+        # restart_events.json（完整重启事件历史）
+        _add_json("restart_events.json", {
+            "exported_at": now.isoformat(),
+            "retention_days": cfg.history_retention_days,
+            "count": len(restart_events),
+            "events": restart_events,
+        })
+
+        # scan_tasks.json / scan_findings.json / engine_errors.json
+        _add_json("scan_tasks.json", scan_tasks)
+        _add_json("scan_findings.json", scan_findings)
+        _add_json("engine_errors.json", engine_errors)
+
+        # policy_snapshot.json
+        _add_json("policy_snapshot.json", policy_snapshot)
+
+        # engine_health.json
+        _add_json("engine_health.json", scanner.health())
+
+        # containers/*.log / *.meta.json（当前快照）
+        for c in containers:
+            if c in docker_logs_out:
+                _add_text(f"containers/{c}.log", docker_logs_out[c])
+            if c in container_meta:
+                _add_json(f"containers/{c}.meta.json", container_meta[c])
+
+        # diagnostics_logs/（持久化目录下所有 .log — timer 快照 + export 快照 + restart 快照）
+        try:
+            from app.diagnostic.storage import _logs_dir
+            logs_dir = _logs_dir()
+            if cfg.storage_enabled and logs_dir.exists():
+                for fpath in logs_dir.glob("*.log"):
+                    try:
+                        tar.add(
+                            str(fpath),
+                            arcname=f"diagnostics_logs/{fpath.name}",
+                        )
+                    except Exception:  # noqa: BLE001
+                        pass
+        except Exception:  # noqa: BLE001
+            # 回退：只用 export 生成的快照
+            for fpath in snapshot_files:
+                try:
+                    tar.add(str(fpath), arcname=f"diagnostics_logs/{fpath.name}")
+                except Exception:
+                    pass
+
+    return buf.getvalue()