feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交
核心功能: - 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制 - FastAPI + PostgreSQL + Redis + Docker Compose 架构 - 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性 - 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页) - 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min) 并发与稳定性修复: - 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行) - JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写 - 移除 SQLite fallback, 统一 PostgreSQL - API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
383
app/engines/extractor.py
Normal file
383
app/engines/extractor.py
Normal file
@@ -0,0 +1,383 @@
|
||||
"""递归安全解包引擎。
|
||||
|
||||
安全约束（全部强制执行，不依赖调用方自觉）：
|
||||
- 解包深度上限
|
||||
- 解压总大小上限与单文件大小上限
|
||||
- 压缩比上限（防压缩炸弹）
|
||||
- 路径穿越拦截（Zip Slip）
|
||||
- 符号链接/硬链接逃逸拦截
|
||||
- 解包目标固定在独立临时目录，文件名重写为安全形式（不直接使用归档内原始名）
|
||||
|
||||
解包出来的文件会加入待检测队列，实现"递归检测"，避免只扫外层被绕过。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from app.engines.base import Finding, ScanTarget, Severity
|
||||
|
||||
ARCHIVE_EXTS = {".zip", ".jar", ".war", ".apk", ".epub", ".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp"}
|
||||
TAR_EXTS = {".tar", ".gz", ".tgz", ".bz2", ".xz", ".tbz2", ".txz"}
|
||||
|
||||
_SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._一-鿿-]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionReport:
|
||||
"""解包结果。"""
|
||||
|
||||
files: List[ScanTarget] = field(default_factory=list)
|
||||
findings: List[Finding] = field(default_factory=list)
|
||||
total_bytes: int = 0
|
||||
truncated: bool = False
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"file_count": len(self.files),
|
||||
"total_bytes": self.total_bytes,
|
||||
"truncated": self.truncated,
|
||||
}
|
||||
|
||||
|
||||
class SafeExtractor:
|
||||
"""递归安全解包器。"""
|
||||
|
||||
def __init__(self, ctx: Any) -> None:
|
||||
self.ctx = ctx
|
||||
self.settings = ctx.settings
|
||||
self.engine_name = "extractor"
|
||||
|
||||
def extract(self, target: ScanTarget, workdir: str) -> ExtractionReport:
|
||||
report = ExtractionReport()
|
||||
ext = (target.declared_ext or "").lower()
|
||||
|
||||
if ext in ARCHIVE_EXTS or zipfile.is_zipfile(target.path):
|
||||
self._extract_zip(target, workdir, report)
|
||||
elif ext in TAR_EXTS or tarfile.is_tarfile(target.path):
|
||||
self._extract_tar(target, workdir, report)
|
||||
|
||||
return report
|
||||
|
||||
# ------------------------------------------------------------------ 工具
|
||||
|
||||
def _sanitize_name(self, raw_name: str) -> str:
|
||||
"""把归档内原始文件名改写为安全的相对路径。
|
||||
|
||||
去掉目录层级与危险字符，避免原始名称中的 ``..``、绝对路径、
|
||||
Windows 保留设备名等造成问题。原始名称仍保留在逻辑路径中以便追溯。
|
||||
"""
|
||||
name = raw_name.replace("\\", "/")
|
||||
base = os.path.basename(name)
|
||||
base = _SAFE_NAME_RE.sub("_", base).strip("._")
|
||||
if not base:
|
||||
base = "unnamed"
|
||||
# 防止 Windows 保留设备名
|
||||
stem, dot, suffix = base.partition(".")
|
||||
reserved = {
|
||||
"con", "prn", "aux", "nul",
|
||||
*(f"com{i}" for i in range(1, 10)),
|
||||
*(f"lpt{i}" for i in range(1, 10)),
|
||||
}
|
||||
if stem.lower() in reserved:
|
||||
stem = f"_{stem}"
|
||||
return f"{stem}{dot}{suffix}" if dot else stem
|
||||
|
||||
def _target_dir(self, workdir: str, target: ScanTarget) -> str:
|
||||
"""为每个文件分配独立的解包子目录。"""
|
||||
digest = abs(hash(target.logical_path or target.filename)) % 100000
|
||||
path = os.path.join(workdir, f"ex_{target.depth}_{digest}")
|
||||
os.makedirs(path, exist_ok=True)
|
||||
return path
|
||||
|
||||
def _budget_exhausted(self, report: ExtractionReport) -> bool:
|
||||
return report.total_bytes >= self.settings.max_extract_total_bytes
|
||||
|
||||
def _record_limit(self, report: ExtractionReport, limit: int = 5000) -> bool:
|
||||
if len(report.files) >= limit:
|
||||
report.truncated = True
|
||||
return True
|
||||
return False
|
||||
|
||||
def _add_finding(
|
||||
self,
|
||||
report: ExtractionReport,
|
||||
severity: str,
|
||||
rule: str,
|
||||
detail: str,
|
||||
weight: float = 0.0,
|
||||
) -> None:
|
||||
report.findings.append(
|
||||
Finding(
|
||||
engine=self.engine_name,
|
||||
layer="L0",
|
||||
severity=severity,
|
||||
rule=rule,
|
||||
detail=detail,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ ZIP
|
||||
|
||||
def _extract_zip(self, target: ScanTarget, workdir: str, report: ExtractionReport) -> None:
|
||||
out_dir = self._target_dir(workdir, target)
|
||||
max_ratio = self.settings.max_extract_ratio
|
||||
max_total = self.settings.max_extract_total_bytes
|
||||
max_depth = self.settings.max_extract_depth
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(target.path) as zf:
|
||||
infos = zf.infolist()
|
||||
for info in infos:
|
||||
if target.depth + 1 > max_depth:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractDepthExceeded",
|
||||
f"嵌套层级超过上限 {max_depth}，深层内容未展开",
|
||||
15.0,
|
||||
)
|
||||
report.truncated = True
|
||||
break
|
||||
|
||||
if self._budget_exhausted(report) or self._record_limit(report):
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractBudgetExceeded",
|
||||
"解包数量或体积达到上限，后续内容未展开",
|
||||
10.0,
|
||||
)
|
||||
break
|
||||
|
||||
if info.is_dir():
|
||||
continue
|
||||
|
||||
name = info.filename
|
||||
if _is_traversal(name):
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.CRITICAL.value,
|
||||
"PathTraversalBlocked",
|
||||
f"已拦截路径穿越条目：{name}",
|
||||
50.0,
|
||||
)
|
||||
continue
|
||||
|
||||
# 压缩炸弹判定：按声明大小与实际膨胀比
|
||||
if info.compress_size > 0:
|
||||
ratio = info.file_size / info.compress_size
|
||||
if ratio > max_ratio:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.HIGH.value,
|
||||
"ArchiveBombBlocked",
|
||||
f"成员 {name} 压缩比 {ratio:.1f}:1 超过阈值，已跳过",
|
||||
35.0,
|
||||
)
|
||||
continue
|
||||
|
||||
if info.file_size > self.settings.max_upload_bytes:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 大小超过单文件上限，已跳过",
|
||||
10.0,
|
||||
)
|
||||
continue
|
||||
|
||||
safe_name = self._sanitize_name(name)
|
||||
dest = os.path.join(out_dir, safe_name)
|
||||
try:
|
||||
with zf.open(info) as src, open(dest, "wb") as out:
|
||||
written = 0
|
||||
while True:
|
||||
chunk = src.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > self.settings.max_upload_bytes:
|
||||
out.close()
|
||||
os.remove(dest)
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 实际解压超限，已丢弃",
|
||||
10.0,
|
||||
)
|
||||
break
|
||||
out.write(chunk)
|
||||
else:
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
path=dest,
|
||||
filename=safe_name,
|
||||
declared_ext=os.path.splitext(safe_name)[1].lower(),
|
||||
logical_path=f"{target.logical_path}!/{name}",
|
||||
depth=target.depth + 1,
|
||||
is_extracted=True,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractMemberFailed",
|
||||
f"成员 {name} 解压失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
except zipfile.BadZipFile as exc:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ArchiveBroken",
|
||||
f"ZIP 结构损坏：{exc}",
|
||||
10.0,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractFailed",
|
||||
f"解包失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ TAR
|
||||
|
||||
def _extract_tar(self, target: ScanTarget, workdir: str, report: ExtractionReport) -> None:
|
||||
out_dir = self._target_dir(workdir, target)
|
||||
max_depth = self.settings.max_extract_depth
|
||||
|
||||
try:
|
||||
with tarfile.open(target.path) as tf:
|
||||
for member in tf.getmembers():
|
||||
if target.depth + 1 > max_depth:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractDepthExceeded",
|
||||
f"嵌套层级超过上限 {max_depth}",
|
||||
15.0,
|
||||
)
|
||||
report.truncated = True
|
||||
break
|
||||
|
||||
if self._budget_exhausted(report) or self._record_limit(report):
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractBudgetExceeded",
|
||||
"解包数量或体积达到上限",
|
||||
10.0,
|
||||
)
|
||||
break
|
||||
|
||||
name = member.name
|
||||
if _is_traversal(name) or member.issym() or member.islnk():
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.CRITICAL.value,
|
||||
"PathTraversalBlocked",
|
||||
f"已拦截路径穿越或链接条目：{name}",
|
||||
50.0,
|
||||
)
|
||||
continue
|
||||
|
||||
if not member.isfile():
|
||||
continue
|
||||
|
||||
if member.size > self.settings.max_upload_bytes:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 大小超过单文件上限，已跳过",
|
||||
10.0,
|
||||
)
|
||||
continue
|
||||
|
||||
safe_name = self._sanitize_name(name)
|
||||
dest = os.path.join(out_dir, safe_name)
|
||||
try:
|
||||
src = tf.extractfile(member)
|
||||
if src is None:
|
||||
continue
|
||||
with src, open(dest, "wb") as out:
|
||||
written = 0
|
||||
while True:
|
||||
chunk = src.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > self.settings.max_upload_bytes:
|
||||
break
|
||||
out.write(chunk)
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
path=dest,
|
||||
filename=safe_name,
|
||||
declared_ext=os.path.splitext(safe_name)[1].lower(),
|
||||
logical_path=f"{target.logical_path}!/{name}",
|
||||
depth=target.depth + 1,
|
||||
is_extracted=True,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractMemberFailed",
|
||||
f"成员 {name} 解压失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
except tarfile.TarError as exc:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ArchiveBroken",
|
||||
f"TAR 结构异常：{exc}",
|
||||
10.0,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractFailed",
|
||||
f"解包失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
|
||||
|
||||
def _is_traversal(name: str) -> bool:
|
||||
normalized = name.replace("\\", "/")
|
||||
if normalized.startswith("/"):
|
||||
return True
|
||||
if len(normalized) > 1 and normalized[1] == ":":
|
||||
return True
|
||||
return ".." in normalized.split("/")
|
||||
|
||||
|
||||
def cleanup_dir(path: str) -> None:
|
||||
"""安全删除临时目录。"""
|
||||
try:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def make_workdir(root: str, prefix: str = "scan_") -> str:
|
||||
os.makedirs(root, exist_ok=True)
|
||||
return tempfile.mkdtemp(prefix=prefix, dir=root)
|
||||
Reference in New Issue
Block a user