Files

478 lines
17 KiB
Python
Raw Permalink Normal View History

"""一键导出诊断包(diagnostics-YYYYMMDD-HHMMSS.tar.gz)。
调用方式(HTTP 接口):
GET /api/v1/telemetry/dump → 返回文件下载
打包内容见方案文档 §3.4。
"""
from __future__ import annotations
import io
import json
import os
import subprocess
import tarfile
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from app.config import get_settings
from app.db import SessionLocal
from app.diagnostic.config import get_diag_config
from app.diagnostic.engine import DiagnosticEngine
from app.diagnostic.docker_probe import detect_docker_access
def _sanitize_filename(name: str) -> str:
"""文件名脱敏:保留扩展名 + SHA256 前 8 位。"""
if not name:
return ""
ext = os.path.splitext(name)[1].lower()
# 截取前 20 字符 + hash-like 后缀
return f"file_{hash(name) % 999999:06d}{ext}"
def _sanitize_ip(ip: str) -> str:
if not ip:
return ""
parts = ip.split(".")
if len(parts) == 4:
return f"{'.'.join(parts[:3])}.*"
return ip
def _bj_time(dt) -> Optional[str]:
"""把 DB 里的 UTC datetime(naive 或 aware)转北京时间字符串。"""
if dt is None:
return None
bj = timezone(timedelta(hours=8))
try:
# naive UTC → 加上 tz 再转
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
bj_dt = dt.astimezone(bj)
return bj_dt.strftime("%Y-%m-%d %H:%M:%S")
except Exception:
return str(dt)
def _collect_scan_tasks(days: int = 7, max_rows: int = 10000) -> List[Dict[str, Any]]:
"""拉过去 N 天的 ScanTask,脱敏后返回 dict 列表。有 max_rows 兜底。"""
from app.models import ScanTask
db = SessionLocal()
try:
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
rows = (
db.query(ScanTask)
.filter(ScanTask.created_at >= cutoff)
.order_by(ScanTask.created_at.desc())
.limit(max_rows)
.all()
)
return [
{
"id": r.id,
"filename_sanitized": _sanitize_filename(r.filename),
"declared_ext": r.declared_ext,
"size": r.size,
"sha256": r.sha256[:16],
"channel": r.channel,
"status": r.status,
"mode": r.mode,
"verdict": r.verdict,
"action": r.action,
"score": r.score,
"reason": r.reason[:500] if r.reason else "",
"policy_version": r.policy_version,
"created_at": _bj_time(r.created_at),
"finished_at": _bj_time(r.finished_at),
"duration_ms": r.duration_ms,
}
for r in rows
]
finally:
db.close()
def _collect_scan_findings(days: int = 7, max_rows: int = 50000) -> List[Dict[str, Any]]:
from app.models import ScanFinding
db = SessionLocal()
try:
# ScanFinding 没有 created_at 字段?查 ScanTask 关联时间
from app.models import ScanTask
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
rows = (
db.query(ScanFinding)
.join(ScanTask, ScanFinding.task_id == ScanTask.id)
.filter(ScanTask.created_at >= cutoff)
.order_by(ScanFinding.id.desc())
.limit(max_rows)
.all()
)
return [
{
"task_id": r.task_id,
"engine": r.engine,
"layer": r.layer,
"severity": r.severity,
"rule": r.rule,
"weight": r.weight,
"detail": (r.detail[:2000] if r.detail else ""),
"extracted_path": r.extracted_path,
}
for r in rows
]
finally:
db.close()
def _collect_engine_errors(days: int = 7, max_rows: int = 5000) -> List[Dict[str, Any]]:
from app.models import ScanTask
db = SessionLocal()
try:
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
rows = (
db.query(ScanTask)
.filter(ScanTask.verdict == "error")
.filter(ScanTask.created_at >= cutoff)
.order_by(ScanTask.created_at.desc())
.limit(max_rows)
.all()
)
return [
{
"id": r.id,
"filename_sanitized": _sanitize_filename(r.filename),
"reason": r.reason[:500] if r.reason else "",
"created_at": _bj_time(r.created_at),
}
for r in rows
]
finally:
db.close()
def _collect_policy_snapshot() -> Dict[str, Any]:
"""策略版本 + YARA 规则列表(不含规则原文)。"""
from app.main import policy_store
policy = policy_store.current()
settings = get_settings()
rules_dir = settings.rules_dir
yara_rules: List[str] = []
if os.path.isdir(rules_dir):
yara_rules = sorted(
f for f in os.listdir(rules_dir)
if f.endswith((".yar", ".yara", ".rules"))
)
return {
"version": policy.version,
"sha256": policy.sha256,
"yara_rules_dir": rules_dir,
"yara_rules_count": len(yara_rules),
"yara_rules_list": yara_rules[:50], # 最多 50 个文件名
"global": policy.raw.get("global", {}),
}
def _collect_container_logs(containers: List[str], tail: int) -> Dict[str, str]:
"""用 docker SDK 取日志。"""
from app.diagnostic.docker_probe import get_client
client = get_client()
result: Dict[str, str] = {}
for c in containers:
try:
container = client.containers.get(c)
raw = container.logs(tail=tail)
result[c] = raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
except Exception as e: # noqa: BLE001
result[c] = f"[导出失败: {e}]"
return result
def _collect_container_meta(containers: List[str]) -> Dict[str, Dict[str, Any]]:
"""用 docker SDK 取容器状态元数据(含 uptime / OOM)。"""
from app.diagnostic.docker_probe import get_client
client = get_client()
meta: Dict[str, Dict[str, Any]] = {}
now = datetime.now(timezone.utc)
for c in containers:
try:
container = client.containers.get(c)
attrs = container.attrs or {}
state = attrs.get("State", {})
started_raw = state.get("StartedAt", "")
started_at = _docker_time(started_raw)
uptime_s = (now - started_at).total_seconds() if started_at else -1
meta[c] = {
"status": state.get("Status", "unknown"),
"health": state.get("Health", {}).get("Status", ""),
"restart_count": attrs.get("RestartCount", -1),
"oom_killed": state.get("OOMKilled", False),
"exit_code": state.get("ExitCode", 0),
"started_at": _bj_time(started_at),
"uptime": _fmt_duration(uptime_s),
"uptime_seconds": int(uptime_s) if uptime_s >= 0 else -1,
"image": attrs.get("Config", {}).get("Image", ""),
}
except Exception as e: # noqa: BLE001
meta[c] = {"error": str(e)[:200]}
return meta
def _collect_compose_state() -> Dict[str, Any]:
"""用 docker SDK 列容器(替代 docker compose ps,输出结构化 JSON)。"""
from app.diagnostic.docker_probe import get_client
client = get_client()
now = datetime.now(timezone.utc)
items: List[Dict[str, Any]] = []
try:
for c in client.containers.list(all=True):
attrs = c.attrs or {}
state = attrs.get("State", {})
started_raw = state.get("StartedAt", "")
started_at = _docker_time(started_raw)
uptime_s = (now - started_at).total_seconds() if started_at else -1
items.append({
"name": c.name,
"status": state.get("Status", "unknown"),
"health": state.get("Health", {}).get("Status", ""),
"restart_count": attrs.get("RestartCount", -1),
"oom_killed": state.get("OOMKilled", False),
"exit_code": state.get("ExitCode", 0),
"image": c.image.tags[0] if c.image.tags else "unknown",
"started_at": _bj_time(started_at),
"uptime": _fmt_duration(uptime_s),
})
except Exception as e: # noqa: BLE001
return {"error": str(e)[:300], "containers": []}
return {"containers": items, "count": len(items)}
def _docker_time(raw: str) -> Optional[datetime]:
"""把 docker 返回的 ISO 时间串转 aware UTC datetime。"""
if not raw:
return None
try:
# docker 返回: "2026-10-28T05:00:12.123456789Z" 或 ".123Z"
s = raw.replace("Z", "+00:00")
# 截断纳秒到微秒(Python fromisoformat 不接受纳秒)
if "." in s:
head, frac = s.split(".", 1)
tz_idx = frac.rfind("+")
if tz_idx >= 0:
frac_ns = frac[:tz_idx]
tz = frac[tz_idx:]
frac_ns = (frac_ns + "000000")[:6]
s = head + "." + frac_ns + tz
return datetime.fromisoformat(s)
except Exception:
return None
def _fmt_duration(seconds: float) -> str:
"""把秒数格式化成人类可读的时长。"""
if seconds < 0:
return "未知"
s = int(seconds)
days, rem = divmod(s, 86400)
hours, rem = divmod(rem, 3600)
mins, secs = divmod(rem, 60)
if days > 0:
return f"{days}d {hours}h {mins}m"
if hours > 0:
return f"{hours}h {mins}m"
if mins > 0:
return f"{mins}m {secs}s"
return f"{secs}s"
# ======================================================================
# 主入口:build_diagnostics_package
# ======================================================================
def build_diagnostics_package() -> bytes:
"""生成完整诊断包,返回 tar.gz 字节。"""
started = time.time()
now = datetime.now(timezone.utc)
cfg = get_diag_config()
settings = get_settings()
containers = [
"meshsafe-api", "meshsafe-postgres",
"meshsafe-redis", "meshsafe-clamav", "meshsafe-nginx",
]
# ---- 1. docker 能力 ----
docker_info = detect_docker_access(mode=cfg.docker_mode)
# ---- 2. 导出前先存当前 docker 日志快照(持久化目录里)----
snapshot_files: List[Path] = []
try:
from app.diagnostic.storage import snapshot_container_logs
if cfg.storage_enabled and docker_info["status"] != "none":
snapshot_files = snapshot_container_logs(
containers, cfg.export_log_tail,
filename_prefix="export",
)
except Exception: # noqa: BLE001
snapshot_files = []
# ---- 3. 规则引擎(全量跑一遍)----
db = SessionLocal()
from app.main import scanner, policy_store
eng = DiagnosticEngine(
db=db, scanner=scanner, policy_store=policy_store,
docker_status=docker_info["status"],
docker_log_tail=cfg.docker_log_tail,
)
rule_results = eng.run_all(run_by="export")
db.close()
# ---- 4. 加载规则历史(持久化 JSONL)----
rule_history: List[Dict[str, Any]] = []
restart_events: List[Dict[str, Any]] = []
try:
from app.diagnostic.storage import load_rule_history, load_restart_events
if cfg.storage_enabled:
rule_history = load_rule_history(cfg.history_retention_days)
restart_events = load_restart_events(cfg.history_retention_days)
except Exception: # noqa: BLE001
rule_history = []
restart_events = []
# 按容器聚合重启事件数(用于 rules_report 的 summary)
restart_by_container: Dict[str, int] = {}
for ev in restart_events:
c = ev.get("container", "?")
restart_by_container[c] = restart_by_container.get(c, 0) + 1
# ---- 5. DB 数据(默认过去 7 天,各有上限兜底)----
span_days = getattr(cfg, "export_db_span_days", 7)
scan_tasks = _collect_scan_tasks(
days=span_days, max_rows=getattr(cfg, "export_scan_tasks_max", 10000),
)
scan_findings = _collect_scan_findings(
days=span_days, max_rows=getattr(cfg, "export_scan_findings_max", 50000),
)
engine_errors = _collect_engine_errors(
days=span_days, max_rows=getattr(cfg, "export_engine_errors_max", 5000),
)
policy_snapshot = _collect_policy_snapshot()
# ---- 6. 容器数据(如果 Docker 可用)----
docker_logs_out: Dict[str, str] = {}
container_meta: Dict[str, Dict[str, Any]] = {}
compose_state: Dict[str, Any] = {"containers": [], "error": "Docker 不可用"}
if docker_info["status"] != "none":
docker_logs_out = _collect_container_logs(containers, cfg.export_log_tail)
container_meta = _collect_container_meta(containers)
compose_state = _collect_compose_state()
# ---- 7. 组装 tar.gz ----
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
def _add_json(name: str, obj: Any) -> None:
data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8")
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
def _add_text(name: str, text: str) -> None:
data = text.encode("utf-8")
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
# meta.json
_add_json("meta.json", {
"exported_at": now.isoformat(),
"generated_by": "MeshSafe Diagnostics Kit v1.0",
"meshsafe_version": "1.1.0",
"docker_image": container_meta.get("meshsafe-api", {}).get("image", "unknown"),
"policy_version": policy_snapshot["version"],
"storage_enabled": cfg.storage_enabled,
"rules_history_count": len(rule_history),
"log_snapshots_included": len(snapshot_files),
"export_duration_ms": int((time.time() - started) * 1000),
# 数据范围
"db_span_days": getattr(cfg, "export_db_span_days", 7),
"scan_tasks_count": len(scan_tasks),
"scan_findings_count": len(scan_findings),
"engine_errors_count": len(engine_errors),
})
# docker_detect.json
_add_json("docker_detect.json", docker_info)
# compose_state.json(结构化 JSON)
_add_json("compose_state.json", compose_state)
# rules_report.json(实时全量 + 持久化历史 + 重启事件汇总)
_add_json("rules_report.json", {
"exported_at": now.isoformat(),
"docker_status": docker_info["status"],
"current_run": [r.to_dict() for r in rule_results],
"history_retention_days": cfg.history_retention_days,
"history_count": len(rule_history),
"history": rule_history,
"restart_events_count": len(restart_events),
"restart_events_summary": restart_by_container,
})
# restart_events.json(完整重启事件历史)
_add_json("restart_events.json", {
"exported_at": now.isoformat(),
"retention_days": cfg.history_retention_days,
"count": len(restart_events),
"events": restart_events,
})
# scan_tasks.json / scan_findings.json / engine_errors.json
_add_json("scan_tasks.json", scan_tasks)
_add_json("scan_findings.json", scan_findings)
_add_json("engine_errors.json", engine_errors)
# policy_snapshot.json
_add_json("policy_snapshot.json", policy_snapshot)
# engine_health.json
_add_json("engine_health.json", scanner.health())
# containers/*.log / *.meta.json(当前快照)
for c in containers:
if c in docker_logs_out:
_add_text(f"containers/{c}.log", docker_logs_out[c])
if c in container_meta:
_add_json(f"containers/{c}.meta.json", container_meta[c])
# diagnostics_logs/(持久化目录下所有 .log — timer 快照 + export 快照 + restart 快照)
try:
from app.diagnostic.storage import _logs_dir
logs_dir = _logs_dir()
if cfg.storage_enabled and logs_dir.exists():
for fpath in logs_dir.glob("*.log"):
try:
tar.add(
str(fpath),
arcname=f"diagnostics_logs/{fpath.name}",
)
except Exception: # noqa: BLE001
pass
except Exception: # noqa: BLE001
# 回退:只用 export 生成的快照
for fpath in snapshot_files:
try:
tar.add(str(fpath), arcname=f"diagnostics_logs/{fpath.name}")
except Exception:
pass
return buf.getvalue()