feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交

核心功能:
- 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制
- FastAPI + PostgreSQL + Redis + Docker Compose 架构
- 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性
- 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页)
- 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min)

并发与稳定性修复:
- 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行)
- JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写
- 移除 SQLite fallback, 统一 PostgreSQL
- API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
2026-10-28 19:07:11 +08:00
commit 16f134e328
113 changed files with 24604 additions and 0 deletions

171
app/diagnostic/config.py Normal file
View File

@@ -0,0 +1,171 @@
"""MeshSafe Diagnostics Kit — 配置读取。
所有配置项可被环境变量覆盖(``MESHSAFE_DIAG_*`` 前缀),
优先级:环境变量 > diagnostics.yaml > 默认值。
"""
from __future__ import annotations
import functools
import os
from pathlib import Path
from typing import Any, Dict
import yaml
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_DEFAULT_YAML = _PROJECT_ROOT / "config" / "diagnostics.yaml"
def _load_yaml(path: Path) -> Dict[str, Any]:
"""读取 YAML 配置,文件不存在返回空 dict。"""
try:
with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
return data.get("diagnostics", {}) if isinstance(data, dict) else {}
except FileNotFoundError:
return {}
except Exception: # noqa: BLE001
return {}
def _env_bool(name: str, default: bool) -> bool:
v = os.environ.get(name)
if v is None:
return default
return v.strip().lower() in ("1", "true", "yes", "on")
def _env_int(name: str, default: int) -> int:
v = os.environ.get(name)
if v is None:
return default
try:
return int(v)
except ValueError:
return default
def _env_str(name: str, default: str) -> str:
return os.environ.get(name, default)
@functools.lru_cache(maxsize=1)
def get_diag_config() -> "DiagConfig":
yaml_data = _load_yaml(_DEFAULT_YAML)
return DiagConfig.from_dict(yaml_data)
class DiagConfig:
"""诊断套件配置。"""
def __init__(self, yaml_data: Dict[str, Any]):
# 顶层开关
yaml_enabled = yaml_data.get("enabled", True)
self.enabled = _env_bool(
"MESHSAFE_DIAG_ENABLED", yaml_enabled
)
# Docker 子能力
docker_yaml = yaml_data.get("docker", {})
yaml_docker_mode = docker_yaml.get("enabled", "auto")
self.docker_mode = _env_str(
"MESHSAFE_DIAG_DOCKER", yaml_docker_mode
).lower()
if self.docker_mode not in ("auto", "true", "false"):
self.docker_mode = "auto"
# 定时器周期(秒)
timers_yaml = yaml_data.get("timers", {})
self.error_scan_interval = _env_int(
"MESHSAFE_DIAG_ERROR_INTERVAL",
timers_yaml.get("error_scan_interval", 30),
)
self.container_check_interval = _env_int(
"MESHSAFE_DIAG_CONTAINER_INTERVAL",
timers_yaml.get("container_check_interval", 300),
)
self.db_aggregate_interval = _env_int(
"MESHSAFE_DIAG_DB_INTERVAL",
timers_yaml.get("db_aggregate_interval", 60),
)
# 日志行数
self.docker_log_tail = int(yaml_data.get("docker_log_tail", 500))
self.export_log_tail = int(yaml_data.get("export_log_tail", 2000))
# 导出包数据范围
export_yaml = yaml_data.get("export", {})
self.export_db_span_days = int(export_yaml.get("db_span_days", 7))
self.export_scan_tasks_max = int(export_yaml.get("scan_tasks_max", 10000))
self.export_scan_findings_max = int(export_yaml.get("scan_findings_max", 50000))
self.export_engine_errors_max = int(export_yaml.get("engine_errors_max", 5000))
# 规则阈值
rules_yaml = yaml_data.get("rules", {})
self.error_ratio_threshold = float(
rules_yaml.get("error_ratio_threshold", 0.05)
)
self.clamav_db_stale_days = int(
rules_yaml.get("clamav_db_stale_days", 7)
)
self.reject_ratio_threshold = float(
rules_yaml.get("reject_ratio_threshold", 0.20)
)
self.suspicious_exceeds_malicious = bool(
rules_yaml.get("suspicious_exceeds_malicious", True)
)
# 持久化存储
storage_yaml = yaml_data.get("storage", {})
self.storage_enabled = _env_bool(
"MESHSAFE_DIAG_STORAGE_ENABLED",
storage_yaml.get("enabled", True),
)
self.storage_dir = storage_yaml.get("dir", "/app/data/diagnostics")
self.history_retention_days = int(
storage_yaml.get("history_retention_days", 7)
)
self.log_snapshot_on_timer = bool(
storage_yaml.get("log_snapshot_on_timer", True)
)
self.log_snapshot_tail = int(
storage_yaml.get("log_snapshot_tail", 500)
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DiagConfig":
return cls(data)
def to_dict(self) -> Dict[str, Any]:
return {
"enabled": self.enabled,
"docker_mode": self.docker_mode,
"timers": {
"error_scan_interval": self.error_scan_interval,
"container_check_interval": self.container_check_interval,
"db_aggregate_interval": self.db_aggregate_interval,
},
"docker_log_tail": self.docker_log_tail,
"export_log_tail": self.export_log_tail,
"export": {
"db_span_days": self.export_db_span_days,
"scan_tasks_max": self.export_scan_tasks_max,
"scan_findings_max": self.export_scan_findings_max,
"engine_errors_max": self.export_engine_errors_max,
},
"rules": {
"error_ratio_threshold": self.error_ratio_threshold,
"clamav_db_stale_days": self.clamav_db_stale_days,
"reject_ratio_threshold": self.reject_ratio_threshold,
"suspicious_exceeds_malicious": self.suspicious_exceeds_malicious,
},
"storage": {
"enabled": self.storage_enabled,
"dir": self.storage_dir,
"history_retention_days": self.history_retention_days,
"log_snapshot_on_timer": self.log_snapshot_on_timer,
"log_snapshot_tail": self.log_snapshot_tail,
},
}