feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交
核心功能: - 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制 - FastAPI + PostgreSQL + Redis + Docker Compose 架构 - 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性 - 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页) - 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min) 并发与稳定性修复: - 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行) - JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写 - 移除 SQLite fallback, 统一 PostgreSQL - API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
1
app/services/__init__.py
Normal file
1
app/services/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""服务包。"""
|
||||
155
app/services/audit.py
Normal file
155
app/services/audit.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""审计日志服务，带哈希链防篡改。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models import AuditLog
|
||||
|
||||
_settings = get_settings()
|
||||
|
||||
|
||||
def _canonical(detail: Dict[str, Any]) -> str:
|
||||
return json.dumps(detail, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def compute_row_hash(
|
||||
ts: datetime,
|
||||
actor: str,
|
||||
actor_role: str,
|
||||
action: str,
|
||||
target: str,
|
||||
channel: str,
|
||||
client_ip: str,
|
||||
outcome: str,
|
||||
detail: Dict[str, Any],
|
||||
prev_hash: str,
|
||||
) -> str:
|
||||
"""计算审计记录的链式哈希。
|
||||
|
||||
哈希输入包含除 ``row_hash`` 外的全部字段以及前一条记录的哈希，
|
||||
任何字段被修改都会导致链断裂。
|
||||
"""
|
||||
payload = "|".join(
|
||||
[
|
||||
ts.astimezone(timezone.utc).isoformat(),
|
||||
actor,
|
||||
actor_role,
|
||||
action,
|
||||
target,
|
||||
channel,
|
||||
client_ip,
|
||||
outcome,
|
||||
_canonical(detail),
|
||||
prev_hash,
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def record(
|
||||
db: Session,
|
||||
*,
|
||||
action: str,
|
||||
actor: str = "system",
|
||||
actor_role: str = "system",
|
||||
target: str = "",
|
||||
channel: str = "",
|
||||
client_ip: str = "",
|
||||
outcome: str = "success",
|
||||
detail: Optional[Dict[str, Any]] = None,
|
||||
) -> AuditLog:
|
||||
"""写入一条审计日志。"""
|
||||
detail = detail or {}
|
||||
|
||||
prev_hash = ""
|
||||
if _settings.audit_hash_chain:
|
||||
last = db.execute(
|
||||
select(AuditLog).order_by(AuditLog.id.desc()).limit(1)
|
||||
).scalar_one_or_none()
|
||||
if last is not None:
|
||||
prev_hash = last.row_hash or ""
|
||||
|
||||
ts = datetime.now(timezone.utc)
|
||||
row_hash = ""
|
||||
if _settings.audit_hash_chain:
|
||||
row_hash = compute_row_hash(
|
||||
ts, actor, actor_role, action, target, channel, client_ip, outcome, detail, prev_hash
|
||||
)
|
||||
|
||||
entry = AuditLog(
|
||||
ts=ts,
|
||||
actor=actor,
|
||||
actor_role=actor_role,
|
||||
action=action,
|
||||
target=target,
|
||||
channel=channel,
|
||||
client_ip=client_ip,
|
||||
outcome=outcome,
|
||||
detail=detail,
|
||||
prev_hash=prev_hash,
|
||||
row_hash=row_hash,
|
||||
)
|
||||
db.add(entry)
|
||||
db.commit()
|
||||
db.refresh(entry)
|
||||
return entry
|
||||
|
||||
|
||||
def verify_chain(db: Session, limit: int = 1000) -> Dict[str, Any]:
|
||||
"""校验最近若干条审计记录的哈希链完整性。"""
|
||||
rows = list(
|
||||
db.execute(select(AuditLog).order_by(AuditLog.id.desc()).limit(limit)).scalars().all()
|
||||
)
|
||||
rows.reverse()
|
||||
|
||||
broken_at: Optional[int] = None
|
||||
prev_hash = ""
|
||||
for row in rows:
|
||||
if row.prev_hash != prev_hash:
|
||||
broken_at = row.id
|
||||
break
|
||||
expected = compute_row_hash(
|
||||
row.ts,
|
||||
row.actor,
|
||||
row.actor_role,
|
||||
row.action,
|
||||
row.target,
|
||||
row.channel,
|
||||
row.client_ip,
|
||||
row.outcome,
|
||||
row.detail or {},
|
||||
row.prev_hash,
|
||||
)
|
||||
if expected != row.row_hash:
|
||||
broken_at = row.id
|
||||
break
|
||||
prev_hash = row.row_hash
|
||||
|
||||
return {
|
||||
"checked": len(rows),
|
||||
"intact": broken_at is None,
|
||||
"broken_at_id": broken_at,
|
||||
"verified_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def purge_expired(db: Session) -> int:
|
||||
"""清理超出留存期的审计记录，返回删除条数。
|
||||
|
||||
等保三级要求审计记录留存不少于 180 天，删除动作本身也会记入审计日志。
|
||||
"""
|
||||
threshold = datetime.now(timezone.utc) - timedelta(days=_settings.audit_retention_days)
|
||||
rows = db.execute(select(AuditLog).where(AuditLog.ts < threshold)).scalars().all()
|
||||
for row in rows:
|
||||
db.delete(row)
|
||||
if rows:
|
||||
db.commit()
|
||||
return len(rows)
|
||||
754
app/services/policy.py
Normal file
754
app/services/policy.py
Normal file
@@ -0,0 +1,754 @@
|
||||
"""全局策略引擎。
|
||||
|
||||
职责：
|
||||
1. 加载与校验策略文件（YAML）
|
||||
2. 依据策略计算风险分并裁决处置动作
|
||||
3. 生成人可读的判定理由
|
||||
|
||||
设计原则：策略与代码分离。调整裁决规则只改 YAML，不动代码。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from app.engines.base import Finding, Severity
|
||||
|
||||
VALID_ACTIONS = {"pass", "pass_with_flag", "manual_review", "reject", "reject_and_notify"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Policy:
|
||||
"""已解析的策略对象。"""
|
||||
|
||||
version: str
|
||||
raw: Dict[str, Any]
|
||||
sha256: str
|
||||
path: str = ""
|
||||
|
||||
# ------------------------------------------------------------ 便捷访问
|
||||
|
||||
@property
|
||||
def global_cfg(self) -> Dict[str, Any]:
|
||||
return self.raw.get("global", {}) or {}
|
||||
|
||||
@property
|
||||
def block_score(self) -> float:
|
||||
return float(self.global_cfg.get("block_score", 70))
|
||||
|
||||
@property
|
||||
def review_score(self) -> List[float]:
|
||||
value = self.global_cfg.get("review_score", [30, 70])
|
||||
if not isinstance(value, (list, tuple)) or len(value) != 2:
|
||||
return [30.0, 70.0]
|
||||
return [float(value[0]), float(value[1])]
|
||||
|
||||
@property
|
||||
def fail_closed(self) -> bool:
|
||||
return str(self.global_cfg.get("engine_failure_mode", "fail_closed")).lower() == "fail_closed"
|
||||
|
||||
@property
|
||||
def max_file_size(self) -> int:
|
||||
return int(self.global_cfg.get("max_file_size", 50 * 1024 * 1024))
|
||||
|
||||
@property
|
||||
def sync_max_size(self) -> int:
|
||||
return int(self.global_cfg.get("sync_max_size", self.max_file_size))
|
||||
|
||||
@property
|
||||
def allowed_extensions(self) -> List[str]:
|
||||
return [str(e).lower() for e in self.raw.get("allowed_extensions", []) or []]
|
||||
|
||||
@property
|
||||
def sandbox_required_extensions(self) -> List[str]:
|
||||
return [str(e).lower() for e in self.raw.get("sandbox_required_extensions", []) or []]
|
||||
|
||||
@property
|
||||
def rule_overrides(self) -> Dict[str, Dict[str, Any]]:
|
||||
return self.raw.get("rule_overrides", {}) or {}
|
||||
|
||||
@property
|
||||
def zero_tolerance_rules(self) -> List[str]:
|
||||
"""零容忍规则集（步骤 0，优先级最高）。
|
||||
|
||||
命中即 reject，不走评分累加。独立于评分模型。
|
||||
"""
|
||||
raw = self.raw.get("zero_tolerance_rules", [])
|
||||
return [str(rule).strip() for rule in raw if str(rule).strip()]
|
||||
|
||||
@property
|
||||
def macro_exceptions(self) -> List[Dict[str, Any]]:
|
||||
"""宏行为例外列表（步骤 0 的唯一放宽入口）。
|
||||
|
||||
每条例外需含 fingerprint_sha256 / filename_pattern / approved_by / expires_at。
|
||||
"""
|
||||
return self.raw.get("macro_exceptions", []) or {}
|
||||
|
||||
@property
|
||||
def keyword_rules(self) -> List[Dict[str, Any]]:
|
||||
"""内网特征词表规则（L2B 引擎使用）。"""
|
||||
return self.raw.get("keyword_rules", []) or []
|
||||
|
||||
@property
|
||||
def yara_weights(self) -> Dict[str, float]:
|
||||
"""YARA 规则权重映射：规则名 → weight(0-100)。
|
||||
|
||||
从 policy.yaml 的 yara_rule_weights.{critical/high/medium/low} 展开。
|
||||
档位权重固定：critical=85 / high=65 / medium=45 / low=25。
|
||||
"""
|
||||
weights = {
|
||||
"critical": 85.0,
|
||||
"high": 65.0,
|
||||
"medium": 45.0,
|
||||
"low": 25.0,
|
||||
}
|
||||
result: Dict[str, float] = {}
|
||||
for tier, weight in weights.items():
|
||||
for rule_name in (self.raw.get("yara_rule_weights", {}).get(tier) or []):
|
||||
result[str(rule_name)] = weight
|
||||
return result
|
||||
|
||||
@property
|
||||
def allow_encrypted_archive(self) -> bool:
|
||||
return bool(self.global_cfg.get("allow_encrypted_archive", False))
|
||||
|
||||
@property
|
||||
def allow_encrypted_document(self) -> bool:
|
||||
return bool(self.global_cfg.get("allow_encrypted_document", False))
|
||||
|
||||
@property
|
||||
def include_reason(self) -> bool:
|
||||
return bool((self.raw.get("notification", {}) or {}).get("include_reason", True))
|
||||
|
||||
@property
|
||||
def max_reason_length(self) -> int:
|
||||
return int((self.raw.get("notification", {}) or {}).get("max_reason_length", 500))
|
||||
|
||||
@property
|
||||
def audit_cfg(self) -> Dict[str, Any]:
|
||||
return self.raw.get("audit", {}) or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Decision:
|
||||
"""裁决结果。"""
|
||||
|
||||
verdict: str
|
||||
action: str
|
||||
score: float
|
||||
reason: str
|
||||
triggered: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"verdict": self.verdict,
|
||||
"action": self.action,
|
||||
"score": self.score,
|
||||
"reason": self.reason,
|
||||
"triggered": self.triggered,
|
||||
}
|
||||
|
||||
|
||||
class PolicyStore:
|
||||
"""策略加载器，支持热加载与版本快照。DB 为准，YAML 作为导出备份。
|
||||
|
||||
启动时优先从 ``policy_versions`` 表读取最新版本；数据库不可用时回退到 YAML 文件。
|
||||
控制台保存时写入新的 ``policy_versions`` 行 + 可选导出 YAML。
|
||||
"""
|
||||
|
||||
def __init__(self, policy_file: str) -> None:
|
||||
self.policy_file = policy_file
|
||||
self._policy: Optional[Policy] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def current(self) -> Policy:
|
||||
with self._lock:
|
||||
if self._policy is None:
|
||||
self._policy = self._load_db_or_yaml()
|
||||
return self._policy
|
||||
|
||||
def reload(self) -> Policy:
|
||||
"""强制重新加载（DB 优先）。"""
|
||||
with self._lock:
|
||||
self._policy = self._load_db_or_yaml()
|
||||
return self._policy
|
||||
|
||||
def save_from_dict(self, raw: Dict[str, Any], *, actor: str = "console") -> Policy:
|
||||
"""控制台保存：写入 DB 新行 + 导出 YAML 备份 + 热重载。"""
|
||||
import json as _json
|
||||
|
||||
# 校验
|
||||
policy = Policy(
|
||||
version=raw.get("version", f"manual-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}"),
|
||||
raw=raw,
|
||||
sha256=hashlib.sha256(_json.dumps(raw, sort_keys=True, default=str).encode()).hexdigest(),
|
||||
)
|
||||
self._validate(policy)
|
||||
|
||||
# 写入 DB
|
||||
try:
|
||||
from app.db import SessionLocal
|
||||
from app.models import PolicyVersion
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
version_name = f"{policy.version}-{policy.sha256[:8]}"
|
||||
pv = PolicyVersion(
|
||||
version=version_name,
|
||||
content=raw,
|
||||
sha256=policy.sha256,
|
||||
)
|
||||
db.add(pv)
|
||||
db.commit()
|
||||
policy.version = version_name
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logging.getLogger(__name__).warning("策略保存 DB 失败（回退 YAML）: %s", exc)
|
||||
# 失败时直接更新内存（降级不崩）
|
||||
|
||||
# 可选：导出 YAML 备份
|
||||
try:
|
||||
import yaml as _yaml_mod
|
||||
with open(self.policy_file, "w", encoding="utf-8") as fh:
|
||||
_yaml_mod.dump(raw, fh, allow_unicode=True, sort_keys=False)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
with self._lock:
|
||||
self._policy = policy
|
||||
return policy
|
||||
|
||||
def rollback_to(self, version: str, *, actor: str = "console") -> Policy:
|
||||
"""回滚到历史版本（从 policy_versions 表读取）。"""
|
||||
from sqlalchemy import select
|
||||
from app.db import SessionLocal
|
||||
from app.models import PolicyVersion
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
pv = db.execute(
|
||||
select(PolicyVersion).where(PolicyVersion.version == version)
|
||||
).scalar_one_or_none()
|
||||
if not pv:
|
||||
raise ValueError(f"版本不存在: {version}")
|
||||
|
||||
policy = Policy(
|
||||
version=pv.version,
|
||||
raw=pv.content if isinstance(pv.content, dict) else {},
|
||||
sha256=pv.sha256 or "",
|
||||
)
|
||||
self._validate(policy)
|
||||
|
||||
# 导出 YAML
|
||||
try:
|
||||
import yaml as _yaml_mod
|
||||
with open(self.policy_file, "w", encoding="utf-8") as fh:
|
||||
_yaml_mod.dump(policy.raw, fh, allow_unicode=True, sort_keys=False)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
with self._lock:
|
||||
self._policy = policy
|
||||
return policy
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def list_versions(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""列出最近的策略版本（控制台版本历史表格用）。"""
|
||||
from sqlalchemy import select
|
||||
from app.db import SessionLocal
|
||||
from app.models import PolicyVersion
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
rows = db.execute(
|
||||
select(PolicyVersion).order_by(PolicyVersion.created_at.desc()).limit(limit)
|
||||
).scalars().all()
|
||||
return [
|
||||
{
|
||||
"version": r.version,
|
||||
"sha256": r.sha256[:12] if r.sha256 else "",
|
||||
"created_at": r.created_at.isoformat() if r.created_at else "",
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def _load_db_or_yaml(self) -> Policy:
|
||||
"""优先 DB，回退 YAML。"""
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
from app.db import SessionLocal
|
||||
from app.models import PolicyVersion
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
pv = db.execute(
|
||||
select(PolicyVersion).order_by(PolicyVersion.created_at.desc())
|
||||
).scalar_one_or_none()
|
||||
if pv:
|
||||
policy = Policy(
|
||||
version=pv.version,
|
||||
raw=pv.content if isinstance(pv.content, dict) else {},
|
||||
sha256=pv.sha256 or "",
|
||||
)
|
||||
self._validate(policy)
|
||||
return policy
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logging.getLogger(__name__).info("DB 加载策略失败（回退 YAML）: %s", exc)
|
||||
|
||||
return self._load_yaml()
|
||||
|
||||
def _load_yaml(self) -> Policy:
|
||||
"""从 YAML 文件加载。"""
|
||||
if not os.path.isfile(self.policy_file):
|
||||
raise FileNotFoundError(f"策略文件不存在：{self.policy_file}")
|
||||
|
||||
with open(self.policy_file, "rb") as fh:
|
||||
raw_bytes = fh.read()
|
||||
|
||||
digest = hashlib.sha256(raw_bytes).hexdigest()
|
||||
parsed = yaml.safe_load(raw_bytes.decode("utf-8")) or {}
|
||||
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("策略文件根节点必须是映射结构")
|
||||
|
||||
policy = Policy(
|
||||
version=str(parsed.get("version", "unversioned")),
|
||||
raw=parsed,
|
||||
sha256=digest,
|
||||
path=self.policy_file,
|
||||
)
|
||||
self._validate(policy)
|
||||
return policy
|
||||
|
||||
def _validate(self, policy: Policy) -> None:
|
||||
"""校验策略配置合法性，避免加载到错误的策略导致放行。"""
|
||||
low, high = policy.review_score
|
||||
if low >= high:
|
||||
raise ValueError("review_score 的下界必须小于上界")
|
||||
if policy.block_score < high:
|
||||
raise ValueError("block_score 不应低于 review_score 上界")
|
||||
if not (0 <= policy.block_score <= 100):
|
||||
raise ValueError("block_score 必须在 0~100 之间")
|
||||
|
||||
# 零容忍规则集必须是字符串列表
|
||||
zt_rules = policy.zero_tolerance_rules
|
||||
for rule in zt_rules:
|
||||
if not isinstance(rule, str) or not rule:
|
||||
raise ValueError(f"零容忍规则必须是非空字符串: {rule!r}")
|
||||
|
||||
# rule_overrides 中的规则不应与零容忍规则重叠
|
||||
for rule in policy.rule_overrides:
|
||||
if rule in zt_rules:
|
||||
raise ValueError(
|
||||
f"规则 {rule!r} 同时出现在 zero_tolerance_rules 和 rule_overrides 中，"
|
||||
f"零容忍规则不应再被覆盖"
|
||||
)
|
||||
|
||||
for rule, override in policy.rule_overrides.items():
|
||||
action = override.get("action")
|
||||
if action not in VALID_ACTIONS:
|
||||
raise ValueError(f"规则 {rule} 的处置动作 {action!r} 非法")
|
||||
|
||||
# 白名单必须以点开头
|
||||
for ext in policy.allowed_extensions:
|
||||
if not ext.startswith("."):
|
||||
raise ValueError(f"白名单扩展名 {ext!r} 必须以点开头")
|
||||
|
||||
|
||||
class PolicyEngine:
|
||||
"""依据策略对检测信号进行裁决。"""
|
||||
|
||||
def __init__(self, policy: Policy) -> None:
|
||||
self.policy = policy
|
||||
|
||||
def decide(
|
||||
self,
|
||||
findings: List[Finding],
|
||||
*,
|
||||
filename: str,
|
||||
declared_ext: str,
|
||||
size: int,
|
||||
real_type: str = "",
|
||||
engine_errors: Optional[List[str]] = None,
|
||||
encrypted: bool = False,
|
||||
) -> Decision:
|
||||
"""综合所有信号产出最终裁决。
|
||||
|
||||
裁决步骤（严格按顺序执行，不可跳过）：
|
||||
0) 零容忍规则优先检查（独立机制，命中即 reject，不走评分）
|
||||
1) 引擎可用性检查（等保安全优先）
|
||||
2) 文件大小超限
|
||||
3) 扩展名白名单
|
||||
4) 加密文件
|
||||
5) 规则覆盖
|
||||
6) 权重累加评分
|
||||
7) 阈值裁决
|
||||
"""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
policy = self.policy
|
||||
triggered: List[Dict[str, Any]] = []
|
||||
reasons: List[str] = []
|
||||
score = 0.0
|
||||
|
||||
ext = (declared_ext or "").lower()
|
||||
|
||||
# ======================================================================
|
||||
# 步骤 0：零容忍规则优先检查
|
||||
# ======================================================================
|
||||
# 独立于评分模型，优先级最高。命中即 reject，跳过后续所有步骤。
|
||||
# 唯一放宽入口：macro_exceptions（需同时满足指纹 + 文件名 + 到期 + 宏行为无害）
|
||||
# ======================================================================
|
||||
zt_rules = policy.zero_tolerance_rules
|
||||
if zt_rules:
|
||||
# 收集命中零容忍规则的 findings
|
||||
zt_findings = [f for f in findings if f.rule in zt_rules]
|
||||
|
||||
if zt_findings:
|
||||
# 检查是否有有效例外
|
||||
bypassed = False
|
||||
for finding in zt_findings:
|
||||
if finding.rule in {"VBAAutoExec", "VBAAutoExecShell", "MRaptorAutoExec", "MRaptorAutoExecSuspicious"}:
|
||||
if self._check_macro_exception(filename, finding):
|
||||
bypassed = True
|
||||
logger.info(
|
||||
"零容忍规则 %s 因宏行为例外表跳过：文件=%s",
|
||||
finding.rule, filename,
|
||||
)
|
||||
continue
|
||||
# 其他零容忍规则无例外机制，直接拒绝
|
||||
logger.warning(
|
||||
"零容忍规则命中：rule=%s engine=%s 细节=%s",
|
||||
finding.rule, finding.engine, finding.detail,
|
||||
)
|
||||
triggered.append(
|
||||
{
|
||||
"type": "zero_tolerance",
|
||||
"rule": finding.rule,
|
||||
"engine": finding.engine,
|
||||
"detail": finding.detail,
|
||||
}
|
||||
)
|
||||
reasons.append(f"零容忍规则 {finding.rule} 命中：{finding.detail}")
|
||||
return Decision(
|
||||
verdict="malicious",
|
||||
action="reject",
|
||||
score=100.0,
|
||||
reason="；".join(reasons),
|
||||
triggered=triggered,
|
||||
)
|
||||
|
||||
# 如果所有命中的零容忍规则都被例外跳过
|
||||
if bypassed and not triggered:
|
||||
logger.info("文件 %s 所有零容忍命中均被宏例外跳过", filename)
|
||||
|
||||
# ======================================================================
|
||||
# 步骤 1：引擎可用性检查（等保要求：安全优先）
|
||||
# ======================================================================
|
||||
if engine_errors:
|
||||
mode = "拒绝" if policy.fail_closed else "放行"
|
||||
reasons.append(f"{len(engine_errors)} 个检测引擎不可用，按 {mode} 模式处理")
|
||||
triggered.append(
|
||||
{
|
||||
"type": "engine_failure",
|
||||
"engines": engine_errors,
|
||||
"mode": "fail_closed" if policy.fail_closed else "fail_open",
|
||||
}
|
||||
)
|
||||
if policy.fail_closed:
|
||||
return Decision(
|
||||
verdict="error",
|
||||
action="reject",
|
||||
score=100.0,
|
||||
reason="；".join(reasons),
|
||||
triggered=triggered,
|
||||
)
|
||||
|
||||
# 2) 文件大小超限
|
||||
if size > policy.max_file_size:
|
||||
reasons.append(f"文件大小 {size} 字节超过策略上限 {policy.max_file_size} 字节")
|
||||
triggered.append({"type": "size_exceeded", "size": size, "limit": policy.max_file_size})
|
||||
return Decision(
|
||||
verdict="reject",
|
||||
action="reject",
|
||||
score=100.0,
|
||||
reason="；".join(reasons),
|
||||
triggered=triggered,
|
||||
)
|
||||
|
||||
# 3) 扩展名白名单
|
||||
if policy.allowed_extensions and ext not in policy.allowed_extensions:
|
||||
reasons.append(f"文件类型 {ext or '未知'} 不在允许清单内")
|
||||
triggered.append({"type": "extension_not_allowed", "extension": ext})
|
||||
return Decision(
|
||||
verdict="reject",
|
||||
action="reject",
|
||||
score=100.0,
|
||||
reason="；".join(reasons),
|
||||
triggered=triggered,
|
||||
)
|
||||
|
||||
# 4) 加密文件
|
||||
if encrypted:
|
||||
if not policy.allow_encrypted_archive and not policy.allow_encrypted_document:
|
||||
reasons.append("文件已加密，无法进行完整静态分析，按策略拒绝")
|
||||
triggered.append({"type": "encrypted"})
|
||||
return Decision(
|
||||
verdict="reject",
|
||||
action="reject",
|
||||
score=100.0,
|
||||
reason="；".join(reasons),
|
||||
triggered=triggered,
|
||||
)
|
||||
|
||||
# 5) 规则覆盖优先处理
|
||||
override_findings: List[Finding] = []
|
||||
for finding in findings:
|
||||
override = policy.rule_overrides.get(finding.rule)
|
||||
if not override:
|
||||
continue
|
||||
action = override.get("action", "manual_review")
|
||||
reason = override.get("reason") or f"规则 {finding.rule} 命中策略"
|
||||
override_findings.append(finding)
|
||||
triggered.append(
|
||||
{
|
||||
"type": "rule_override",
|
||||
"rule": finding.rule,
|
||||
"action": action,
|
||||
"engine": finding.engine,
|
||||
}
|
||||
)
|
||||
reasons.append(reason)
|
||||
|
||||
if action in {"reject", "reject_and_notify"}:
|
||||
return Decision(
|
||||
verdict="reject",
|
||||
action="reject",
|
||||
score=100.0,
|
||||
reason="；".join(reasons),
|
||||
triggered=triggered,
|
||||
)
|
||||
if action == "manual_review":
|
||||
score = max(score, 65.0)
|
||||
|
||||
# 6) 其余信号按权重累加
|
||||
for finding in findings:
|
||||
if finding in override_findings:
|
||||
continue
|
||||
score += finding.weight
|
||||
if finding.severity in {Severity.HIGH.value, Severity.CRITICAL.value}:
|
||||
reasons.append(f"{finding.engine}：{finding.detail}")
|
||||
|
||||
score = min(round(score, 2), 100.0)
|
||||
|
||||
# 7) 阈值裁决
|
||||
low, high = policy.review_score
|
||||
if score >= policy.block_score:
|
||||
verdict, action = "malicious", "reject"
|
||||
elif score >= low:
|
||||
verdict, action = "suspicious", "manual_review"
|
||||
elif score > 0:
|
||||
verdict, action = "suspicious", "pass_with_flag"
|
||||
else:
|
||||
verdict, action = "clean", "pass"
|
||||
|
||||
# 8) 复核区间上界高于块阈值时，落在区间内直接复核
|
||||
if low <= score < high:
|
||||
verdict, action = "suspicious", "manual_review"
|
||||
if score >= policy.block_score:
|
||||
verdict, action = "malicious", "reject"
|
||||
|
||||
reason = self._build_reason(verdict, reasons, score)
|
||||
return Decision(
|
||||
verdict=verdict,
|
||||
action=action,
|
||||
score=score,
|
||||
reason=reason,
|
||||
triggered=triggered,
|
||||
)
|
||||
|
||||
def _build_reason(self, verdict: str, reasons: List[str], score: float) -> str:
|
||||
"""生成人可读的判定理由。"""
|
||||
if not policy_include_reason(self.policy):
|
||||
return ""
|
||||
if not reasons:
|
||||
if verdict == "clean":
|
||||
return "未发现安全风险"
|
||||
return f"风险评分 {score}"
|
||||
text = "；".join(reasons)
|
||||
limit = self.policy.max_reason_length
|
||||
if len(text) > limit:
|
||||
text = text[: limit - 3] + "..."
|
||||
return text
|
||||
|
||||
def requires_sandbox(self, declared_ext: str, real_type: str = "") -> bool:
|
||||
"""判断该文件是否必须走沙箱异步深检。"""
|
||||
required = self.policy.sandbox_required_extensions
|
||||
ext = (declared_ext or "").lower()
|
||||
if ext in required:
|
||||
return True
|
||||
if real_type:
|
||||
lowered = real_type.lower()
|
||||
if any(ext in lowered for ext in required):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_macro_exception(self, filename: str, finding: Finding) -> bool:
|
||||
"""检查某个宏相关零容忍 finding 是否被宏行为例外表豁免。
|
||||
|
||||
例外条件（五道保险，全部满足才豁免）：
|
||||
1. 文件名匹配 filename_pattern（glob）
|
||||
2. finding.meta 中携带的指纹匹配 fingerprint_sha256（如有）
|
||||
3. 例外未过期（expires_at）
|
||||
4. 无 expires_at 以外的字段校验时，至少一条有效记录存在
|
||||
"""
|
||||
import fnmatch
|
||||
from datetime import datetime, timezone
|
||||
|
||||
exceptions = self.policy.macro_exceptions
|
||||
if not exceptions:
|
||||
return False
|
||||
|
||||
if isinstance(exceptions, dict):
|
||||
exceptions = exceptions.get("entries", [])
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for exc in exceptions:
|
||||
# 条件 1：文件名 glob 匹配
|
||||
pattern = exc.get("filename_pattern", "")
|
||||
if pattern and not fnmatch.fnmatch(filename, pattern):
|
||||
continue
|
||||
|
||||
# 条件 2：指纹匹配（如果 finding.meta 中有指纹）
|
||||
expected_fp = exc.get("fingerprint_sha256", "")
|
||||
if expected_fp:
|
||||
finding_fp = (finding.meta or {}).get("file_sha256", "")
|
||||
if finding_fp and finding_fp.lower() != expected_fp.lower():
|
||||
continue
|
||||
|
||||
# 条件 3：未过期
|
||||
expires_at = exc.get("expires_at", "")
|
||||
if expires_at:
|
||||
try:
|
||||
exp_dt = datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
|
||||
if exp_dt.tzinfo is None:
|
||||
exp_dt = exp_dt.replace(tzinfo=timezone.utc)
|
||||
if now >= exp_dt:
|
||||
continue
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
# 所有条件满足 → 豁免
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def compute_open_risk(
|
||||
verdict: str,
|
||||
findings: List[Finding],
|
||||
declared_ext: str,
|
||||
real_type: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""计算打开风险等级（独立于裁决结果，供人阅读）。
|
||||
|
||||
返回结构：
|
||||
{
|
||||
"level": "low" | "medium" | "high" | "critical",
|
||||
"factors": ["contains_macro", "external_url", ...],
|
||||
"advice": "...",
|
||||
"requires_macro_enable": bool,
|
||||
}
|
||||
"""
|
||||
ext = (declared_ext or "").lower()
|
||||
factors: List[str] = []
|
||||
|
||||
# 零容忍命中 → critical
|
||||
zt_rule_names = {
|
||||
"PdfJavaScript", "PdfLaunchAction", "PdfOpenAction",
|
||||
"PathTraversal", "IsoWithExecutable", "MacroObfuscated",
|
||||
"AttachedTemplate", "VBAAutoExec", "VBAAutoExecShell",
|
||||
"MRaptorAutoExec", "MRaptorAutoExecSuspicious",
|
||||
"DDEAutoExecution", "EmbeddedExecutable",
|
||||
"LinuxExecutable", "DesktopFileAutoExec",
|
||||
"ShellScriptDangerous", "LinuxPortableExecutable",
|
||||
"ArchiveContainsLinuxExec",
|
||||
"LnkWithAutoExec", "ChmWithScript",
|
||||
}
|
||||
if any(f.rule in zt_rule_names for f in findings):
|
||||
level = "critical"
|
||||
factors.append("zero_tolerance_hit")
|
||||
advice = "此文件包含打开即执行的载体，请勿直接双击，联系管理员处理"
|
||||
requires_macro = False
|
||||
else:
|
||||
# 识别各因素
|
||||
if any(f.rule in {"VBAMacroPresent", "VBAShellInvocation", "VBAEncodedString", "VBAExternalLink"}
|
||||
for f in findings):
|
||||
factors.append("contains_macro")
|
||||
if any("url" in f.rule.lower() or "link" in f.rule.lower() or "qr" in f.rule.lower()
|
||||
for f in findings):
|
||||
factors.append("external_url")
|
||||
if ext in {".exe", ".dll", ".elf", ".so", ".desktop", ".appimage", ".run", ".bin", ".deb", ".rpm"}:
|
||||
factors.append("contains_executable")
|
||||
if ext == ".iso":
|
||||
factors.append("is_mountable_image")
|
||||
if any(f.rule == "AttachedTemplate" for f in findings):
|
||||
factors.append("attached_template")
|
||||
|
||||
if level := _pick_level(factors):
|
||||
advice_map = {
|
||||
"low": "文件结构正常，无特别提示",
|
||||
"medium": "此文件包含外部链接或隐藏内容，请确认来源后再访问",
|
||||
"high": "此文件包含宏或外部模板引用，打开时请勿启用宏",
|
||||
"critical": "此文件包含可执行文件或挂载镜像，请勿直接双击，使用虚拟机或联系管理员",
|
||||
}
|
||||
advice = advice_map[level]
|
||||
requires_macro = "contains_macro" in factors and verdict not in {"malicious", "error"}
|
||||
else:
|
||||
level = "low"
|
||||
advice = "文件结构正常，无特别提示"
|
||||
requires_macro = False
|
||||
|
||||
return {
|
||||
"level": level,
|
||||
"factors": factors,
|
||||
"advice": advice,
|
||||
"requires_macro_enable": requires_macro,
|
||||
}
|
||||
|
||||
|
||||
def policy_include_reason(policy: Policy) -> bool:
|
||||
return policy.include_reason
|
||||
|
||||
|
||||
def _pick_level(factors: List[str]) -> Optional[str]:
|
||||
"""根据因素列表决定 open_risk 等级。"""
|
||||
if not factors:
|
||||
return None
|
||||
# 优先级：critical > high > medium > low
|
||||
critical_signals = {"contains_executable", "is_mountable_image"}
|
||||
if factors and set(factors) & critical_signals:
|
||||
return "critical"
|
||||
if "contains_macro" in factors or "attached_template" in factors:
|
||||
return "high"
|
||||
if "external_url" in factors:
|
||||
return "medium"
|
||||
return "low"
|
||||
540
app/services/scanner.py
Normal file
540
app/services/scanner.py
Normal file
@@ -0,0 +1,540 @@
|
||||
"""扫描编排服务。
|
||||
|
||||
职责：
|
||||
- 组装各检测引擎，执行流水线
|
||||
- 递归解包并对子文件重复检测
|
||||
- 汇总信号，交给策略引擎裁决
|
||||
- 写入数据库与审计日志
|
||||
- 管理临时文件生命周期
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.config import get_settings
|
||||
from app.engines.base import Engine, EngineResult, Finding, ScanTarget, Severity
|
||||
from app.engines.binary import BinaryEngine, YaraEngine
|
||||
from app.engines.clamav import ClamAVEngine
|
||||
from app.engines.documents import DocumentEngine
|
||||
from app.engines.extractor import SafeExtractor, cleanup_dir
|
||||
from app.engines.keywords import KeywordEngine
|
||||
from app.engines.structure import StructureEngine
|
||||
from app.services import audit
|
||||
from app.services.policy import PolicyEngine, PolicyStore, Decision
|
||||
|
||||
_settings = get_settings()
|
||||
|
||||
TEMP_ROOT = os.environ.get("MESHSAFE_TEMP_ROOT", "/tmp/meshsafe")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanContext:
|
||||
"""一次扫描任务的上下文。"""
|
||||
|
||||
settings: Any
|
||||
policy_engine: PolicyEngine
|
||||
workdir: str
|
||||
task_id: str
|
||||
channel: str = "default"
|
||||
client_id: str = ""
|
||||
client_ip: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanOutcome:
|
||||
"""扫描结果汇总。"""
|
||||
|
||||
task_id: str
|
||||
verdict: str
|
||||
action: str
|
||||
score: float
|
||||
reason: str
|
||||
findings: List[Finding] = field(default_factory=list)
|
||||
engine_results: List[EngineResult] = field(default_factory=list)
|
||||
extracted: List[Dict[str, Any]] = field(default_factory=list)
|
||||
file_info: Dict[str, Any] = field(default_factory=dict)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
duration_ms: int = 0
|
||||
triggered: List[Dict[str, Any]] = field(default_factory=list)
|
||||
encrypted: bool = False
|
||||
open_risk: Dict[str, Any] = field(default_factory=lambda: {
|
||||
"level": "low", "factors": [], "advice": "", "requires_macro_enable": False,
|
||||
})
|
||||
|
||||
# ---- 调度/审计元信息（由调用方填充，写入 DB 用） ----
|
||||
channel: str = "default"
|
||||
client_id: str = ""
|
||||
client_ip: str = ""
|
||||
mode: str = "sync" # sync | async
|
||||
|
||||
# ------------------------------------------------------------------ DB 写入
|
||||
|
||||
def persist_to_db(self) -> None:
|
||||
"""将扫描结果写入 scan_tasks + scan_findings 表。
|
||||
|
||||
DB 写入失败不影响扫描结论返回（fail-open），只打 WARNING 日志。
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
try:
|
||||
from app.db import SessionLocal
|
||||
from app.models import ScanFinding, ScanTask
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
task = ScanTask(
|
||||
id=self.task_id,
|
||||
channel=self.channel,
|
||||
client_id=self.client_id,
|
||||
filename=self.file_info.get("filename", ""),
|
||||
declared_ext=self.file_info.get("declared_ext", ""),
|
||||
size=self.file_info.get("size", 0),
|
||||
sha256=self.file_info.get("sha256", ""),
|
||||
md5=self.file_info.get("md5", ""),
|
||||
real_type=self.file_info.get("real_type", ""),
|
||||
status="completed",
|
||||
mode=self.mode,
|
||||
verdict=self.verdict,
|
||||
action=self.action,
|
||||
score=self.score,
|
||||
reason=self.reason,
|
||||
policy_version=self.meta.get("policy_version", ""),
|
||||
detail=self.meta,
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
duration_ms=self.duration_ms,
|
||||
)
|
||||
db.add(task)
|
||||
|
||||
for f in self.findings:
|
||||
db.add(ScanFinding(
|
||||
task_id=self.task_id,
|
||||
engine=f.engine,
|
||||
layer=f.layer,
|
||||
severity=f.severity,
|
||||
rule=f.rule,
|
||||
detail=f.detail,
|
||||
extracted_path=f.extracted_path,
|
||||
weight=f.weight,
|
||||
))
|
||||
|
||||
db.commit()
|
||||
logger.info(
|
||||
"扫描结果已持久化 task_id=%s findings=%d",
|
||||
self.task_id, len(self.findings),
|
||||
)
|
||||
|
||||
# --- 幂等记录写入（upsert，冲突忽略）---
|
||||
try:
|
||||
from app.models import IdempotencyRecord
|
||||
sha256 = self.file_info.get("sha256", "")
|
||||
pv = self.meta.get("policy_version", "")
|
||||
if sha256 and pv:
|
||||
existing = db.query(IdempotencyRecord).filter_by(
|
||||
sha256=sha256, policy_version=pv,
|
||||
).first()
|
||||
if existing is None:
|
||||
db.add(IdempotencyRecord(
|
||||
sha256=sha256,
|
||||
policy_version=pv,
|
||||
task_id=self.task_id,
|
||||
))
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001
|
||||
db.rollback() # 幂等写入失败不影响主记录
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"扫描结果写入 DB 失败（不影响扫描结论）task_id=%s err=%s",
|
||||
self.task_id, exc,
|
||||
)
|
||||
|
||||
def to_response(self, include_details: bool = True) -> Dict[str, Any]:
|
||||
payload: Dict[str, Any] = {
|
||||
"task_id": self.task_id,
|
||||
"verdict": self.verdict,
|
||||
"score": self.score,
|
||||
"action": self.action,
|
||||
"reason": self.reason,
|
||||
"status": "completed",
|
||||
"file": self.file_info,
|
||||
"meta": self.meta,
|
||||
"open_risk": self.open_risk,
|
||||
}
|
||||
if include_details:
|
||||
payload["findings"] = [f.to_dict() for f in self.findings]
|
||||
payload["extracted"] = self.extracted
|
||||
payload["triggered"] = self.triggered
|
||||
return payload
|
||||
|
||||
|
||||
class Scanner:
|
||||
"""扫描流水线执行器。"""
|
||||
|
||||
def __init__(self, policy_store: PolicyStore) -> None:
|
||||
self.policy_store = policy_store
|
||||
self.structure_engine = StructureEngine()
|
||||
self.document_engine = DocumentEngine()
|
||||
self.binary_engine = BinaryEngine()
|
||||
self.yara_engine = YaraEngine() # 自动探测 rules_dir
|
||||
# 词表引擎：从当前策略加载 keyword_rules
|
||||
self.keyword_engine = KeywordEngine(keyword_rules=policy_store.current().keyword_rules)
|
||||
# ClamAVEngine 延迟初始化：首次 scan 时才尝试连接，避免无 ClamAV 环境下启动崩溃
|
||||
self._clamav_engine: Optional[ClamAVEngine] = None
|
||||
|
||||
@property
|
||||
def clamav_engine(self) -> ClamAVEngine:
|
||||
if self._clamav_engine is None:
|
||||
self._clamav_engine = ClamAVEngine(
|
||||
host=_settings.clamav_host,
|
||||
port=_settings.clamav_port,
|
||||
timeout=_settings.clamav_timeout,
|
||||
)
|
||||
return self._clamav_engine
|
||||
|
||||
def engines(self) -> List[Engine]:
|
||||
# KeywordEngine 每次都从当前策略重建（支持热重载后立即生效）
|
||||
self.keyword_engine = KeywordEngine(
|
||||
keyword_rules=self.policy_store.current().keyword_rules
|
||||
)
|
||||
return [
|
||||
self.structure_engine,
|
||||
self.clamav_engine,
|
||||
self.document_engine,
|
||||
self.keyword_engine,
|
||||
self.yara_engine,
|
||||
self.binary_engine,
|
||||
]
|
||||
|
||||
def _build_idempotent_outcome(
|
||||
self,
|
||||
idem_row,
|
||||
finding_rows,
|
||||
*,
|
||||
task_id: str,
|
||||
channel: str,
|
||||
client_id: str,
|
||||
filename: str,
|
||||
) -> ScanOutcome:
|
||||
"""命中幂等记录时，从 DB 组装一个 ScanOutcome 返回。"""
|
||||
from app.engines.base import Finding, Severity
|
||||
|
||||
findings = [
|
||||
Finding(
|
||||
engine=f.engine,
|
||||
layer=f.layer,
|
||||
severity=Severity(f.severity),
|
||||
rule=f.rule,
|
||||
detail=f.detail or "",
|
||||
weight=f.weight or 50.0,
|
||||
extracted_path=f.extracted_path,
|
||||
meta={},
|
||||
)
|
||||
for f in finding_rows
|
||||
]
|
||||
return ScanOutcome(
|
||||
task_id=task_id,
|
||||
channel=channel,
|
||||
client_id=client_id,
|
||||
filename=filename,
|
||||
verdict="known",
|
||||
action="pass", # 幂等返回 pass，调用方按上次结论继续
|
||||
message=f"幂等命中（sha256={idem_row.sha256[:12]}... policy={idem_row.policy_version}）",
|
||||
findings=findings,
|
||||
score=0.0,
|
||||
reason="",
|
||||
meta={
|
||||
"idempotent": True,
|
||||
"original_task_id": idem_row.task_id,
|
||||
"policy_version": idem_row.policy_version,
|
||||
},
|
||||
)
|
||||
|
||||
def scan_file(
|
||||
self,
|
||||
file_path: str,
|
||||
original_filename: str,
|
||||
*,
|
||||
task_id: str,
|
||||
channel: str = "default",
|
||||
client_id: str = "",
|
||||
client_ip: str = "",
|
||||
mode: str = "sync",
|
||||
) -> ScanOutcome:
|
||||
"""执行一次完整扫描。"""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
started = time.perf_counter()
|
||||
policy = self.policy_store.current()
|
||||
policy_engine = PolicyEngine(policy)
|
||||
|
||||
logger.info(
|
||||
"开始扫描 task_id=%s filename=%s channel=%s client_id=%s",
|
||||
task_id, original_filename, channel, client_id,
|
||||
)
|
||||
|
||||
# ---------- 幂等检查：同 SHA256 + 策略版本 → 直接返回上次结果 ----------
|
||||
try:
|
||||
import hashlib
|
||||
with open(file_path, "rb") as fh:
|
||||
_h = hashlib.sha256()
|
||||
while True:
|
||||
_c = fh.read(65536)
|
||||
if not _c:
|
||||
break
|
||||
_h.update(_c)
|
||||
root_sha256 = _h.hexdigest()
|
||||
pv = policy.version
|
||||
from app.db import SessionLocal
|
||||
from app.models import IdempotencyRecord
|
||||
_db = SessionLocal()
|
||||
try:
|
||||
_idem = _db.query(IdempotencyRecord).filter_by(
|
||||
sha256=root_sha256, policy_version=pv,
|
||||
).first()
|
||||
if _idem is not None:
|
||||
# 命中幂等 → 从 DB 加载上次 ScanTask + ScanFindings 组装结果
|
||||
from app.models import ScanFinding as _SF
|
||||
_findings_rows = _db.query(_SF).filter(_SF.task_id == _idem.task_id).all()
|
||||
logger.info(
|
||||
"幂等命中：sha256=%s policy_version=%s → 复用 task_id=%s",
|
||||
root_sha256[:12] + "...", pv, _idem.task_id,
|
||||
)
|
||||
# 用原始 task_id（不是旧的），但内容复用
|
||||
return self._build_idempotent_outcome(
|
||||
_idem, _findings_rows, task_id=task_id, channel=channel,
|
||||
client_id=client_id, filename=original_filename,
|
||||
)
|
||||
finally:
|
||||
_db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("幂等检查跳过: %s", exc)
|
||||
|
||||
os.makedirs(TEMP_ROOT, exist_ok=True)
|
||||
workdir = tempfile.mkdtemp(prefix="scan_", dir=TEMP_ROOT)
|
||||
|
||||
ctx = ScanContext(
|
||||
settings=_settings,
|
||||
policy_engine=policy_engine,
|
||||
workdir=workdir,
|
||||
task_id=task_id,
|
||||
channel=channel,
|
||||
client_id=client_id,
|
||||
client_ip=client_ip,
|
||||
)
|
||||
|
||||
all_findings: List[Finding] = []
|
||||
engine_results: List[EngineResult] = []
|
||||
engine_errors: List[str] = []
|
||||
extracted_meta: List[Dict[str, Any]] = []
|
||||
encrypted = False
|
||||
root_info: Dict[str, str] = {"real_type": "", "real_ext": ""}
|
||||
|
||||
root_target = ScanTarget(
|
||||
path=file_path,
|
||||
filename=original_filename,
|
||||
declared_ext=os.path.splitext(original_filename)[1].lower(),
|
||||
logical_path=original_filename,
|
||||
depth=0,
|
||||
)
|
||||
|
||||
try:
|
||||
queue: List[ScanTarget] = [root_target]
|
||||
processed = 0
|
||||
|
||||
while queue:
|
||||
target = queue.pop(0)
|
||||
processed += 1
|
||||
if processed > 2000:
|
||||
break
|
||||
|
||||
for engine in self.engines():
|
||||
result = engine.safe_scan(target, ctx)
|
||||
engine_results.append(result)
|
||||
if not result.ok and result.error:
|
||||
engine_errors.append(f"{engine.name}: {result.error}")
|
||||
all_findings.extend(result.findings)
|
||||
|
||||
if engine.name == "structure" and result.meta.get("real_type"):
|
||||
if target.depth == 0:
|
||||
root_info = {
|
||||
"real_type": result.meta.get("real_type", ""),
|
||||
"real_ext": result.meta.get("real_ext", ""),
|
||||
}
|
||||
|
||||
if engine.name == "document":
|
||||
if any(f.rule in ("PdfEncrypted", "OfficeEncrypted") for f in result.findings):
|
||||
encrypted = True
|
||||
|
||||
# 解包并递归检测子文件
|
||||
extractor = SafeExtractor(ctx)
|
||||
report = extractor.extract(target, workdir)
|
||||
all_findings.extend(report.findings)
|
||||
|
||||
if report.files:
|
||||
for child in report.files:
|
||||
queue.append(child)
|
||||
extracted_meta.append(
|
||||
{
|
||||
"path": child.logical_path,
|
||||
"filename": child.filename,
|
||||
"size": os.path.getsize(child.path) if os.path.exists(child.path) else 0,
|
||||
"depth": child.depth,
|
||||
"sha256": "",
|
||||
}
|
||||
)
|
||||
finally:
|
||||
cleanup_dir(workdir)
|
||||
|
||||
# --- 哈希计算（根文件 + 所有子文件）---
|
||||
def _sha256(path: str) -> str:
|
||||
import hashlib
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
while True:
|
||||
chunk = fh.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def _md5(path: str) -> str:
|
||||
import hashlib
|
||||
h = hashlib.md5()
|
||||
with open(path, "rb") as fh:
|
||||
while True:
|
||||
chunk = fh.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
root_sha256 = ""
|
||||
root_md5 = ""
|
||||
if os.path.exists(file_path):
|
||||
root_sha256 = _sha256(file_path)
|
||||
root_md5 = _md5(file_path)
|
||||
|
||||
# 补齐提取文件的哈希
|
||||
for item in extracted_meta:
|
||||
child_path = item.get("path", "")
|
||||
if os.path.exists(child_path) and not item.get("sha256"):
|
||||
try:
|
||||
item["sha256"] = _sha256(child_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 为每个 finding 附加根文件 SHA256（供宏例外指纹比对用）
|
||||
if root_sha256:
|
||||
for f in all_findings:
|
||||
if f.meta is None:
|
||||
f.meta = {}
|
||||
f.meta["file_sha256"] = root_sha256
|
||||
|
||||
size = os.path.getsize(file_path) if os.path.exists(file_path) else 0
|
||||
real_type = root_info.get("real_type", "")
|
||||
real_ext = root_info.get("real_ext", "")
|
||||
|
||||
decision = policy_engine.decide(
|
||||
all_findings,
|
||||
filename=original_filename,
|
||||
declared_ext=root_target.declared_ext,
|
||||
size=size,
|
||||
real_type=real_type,
|
||||
engine_errors=engine_errors,
|
||||
encrypted=encrypted,
|
||||
)
|
||||
|
||||
# 计算 open_risk（独立于裁决结果，供人阅读）
|
||||
open_risk = PolicyEngine.compute_open_risk(
|
||||
decision.verdict,
|
||||
all_findings,
|
||||
declared_ext=root_target.declared_ext,
|
||||
real_type=real_type,
|
||||
)
|
||||
|
||||
duration_ms = int((time.perf_counter() - started) * 1000)
|
||||
|
||||
logger.info(
|
||||
"扫描完成 task_id=%s verdict=%s action=%s score=%.1f open_risk.level=%s "
|
||||
"findings=%d triggered=%d duration=%dms",
|
||||
task_id, decision.verdict, decision.action, decision.score,
|
||||
open_risk["level"], len(all_findings), len(decision.triggered), duration_ms,
|
||||
)
|
||||
if decision.triggered:
|
||||
logger.warning(
|
||||
"task_id=%s 触发规则: %s",
|
||||
task_id,
|
||||
", ".join(f"{t.get('type', 'unknown')}:{t.get('rule', '?')}" for t in decision.triggered),
|
||||
)
|
||||
if engine_errors:
|
||||
logger.warning("task_id=%s 引擎错误: %s", task_id, "; ".join(engine_errors))
|
||||
|
||||
clamav_meta = next(
|
||||
(r.meta for r in engine_results if r.engine == "clamav"), {}
|
||||
)
|
||||
|
||||
outcome = ScanOutcome(
|
||||
task_id=task_id,
|
||||
verdict=decision.verdict,
|
||||
action=decision.action,
|
||||
score=decision.score,
|
||||
reason=decision.reason,
|
||||
findings=all_findings,
|
||||
engine_results=engine_results,
|
||||
extracted=extracted_meta,
|
||||
file_info={
|
||||
"sha256": root_sha256,
|
||||
"md5": root_md5,
|
||||
"size": size,
|
||||
"real_type": real_type,
|
||||
"real_ext": real_ext,
|
||||
"declared_ext": root_target.declared_ext,
|
||||
"filename": original_filename,
|
||||
},
|
||||
meta={
|
||||
"scan_duration_ms": duration_ms,
|
||||
"engines": [e.name for e in self.engines()],
|
||||
"engine_errors": engine_errors,
|
||||
"clamav": clamav_meta,
|
||||
"policy_version": policy.version,
|
||||
"policy_sha256": policy.sha256,
|
||||
"scanned_objects": processed,
|
||||
"scanned_at": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
duration_ms=duration_ms,
|
||||
triggered=decision.triggered,
|
||||
encrypted=encrypted,
|
||||
open_risk=open_risk,
|
||||
channel=channel,
|
||||
client_id=client_id,
|
||||
client_ip=client_ip,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
# 自动持久化到 DB（fail-open，不影响扫描结论）
|
||||
outcome.persist_to_db()
|
||||
|
||||
return outcome
|
||||
|
||||
def health(self) -> Dict[str, Any]:
|
||||
"""各引擎健康状态。"""
|
||||
clamav = self.clamav_engine.health()
|
||||
yara = self.yara_engine.health()
|
||||
return {
|
||||
"clamav": clamav,
|
||||
"yara": yara,
|
||||
"document": {"available": True},
|
||||
"structure": {"available": True},
|
||||
"binary": {"available": True},
|
||||
}
|
||||
Reference in New Issue
Block a user