feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交

核心功能:
- 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制
- FastAPI + PostgreSQL + Redis + Docker Compose 架构
- 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性
- 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页)
- 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min)

并发与稳定性修复:
- 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行)
- JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写
- 移除 SQLite fallback, 统一 PostgreSQL
- API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
2026-10-28 19:07:11 +08:00
commit 16f134e328
113 changed files with 24604 additions and 0 deletions

1
app/security/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""安全包。"""

106
app/security/crypto.py Normal file
View File

@@ -0,0 +1,106 @@
"""密码学基础工具:密钥派生、口令哈希、常数时间比较。"""
from __future__ import annotations
import base64
import hashlib
import hmac
import os
from typing import Tuple
# 口令哈希迭代次数。等保三级要求存储口令不可逆,PBKDF2-HMAC-SHA256
# 迭代 60 万次在现代 CPU 上约 0.3~0.6 秒,兼顾抗暴力破解与可用性。
PBKDF2_ITERATIONS = 600_000
PBKDF2_ALGORITHM = "sha256"
SALT_BYTES = 16
def derive_key(secret: str, salt: str, info: str, length: int = 32) -> bytes:
"""使用 HKDF-SHA256 从调用方密钥派生用途隔离的子密钥。
不同用途使用不同 ``info`` 标签,使得请求签名密钥与响应签名密钥相互独立。
"""
prk = hmac.new(
salt.encode("utf-8"), secret.encode("utf-8"), hashlib.sha256
).digest()
okm = b""
block = b""
counter = 1
while len(okm) < length:
block = hmac.new(prk, block + info.encode("utf-8") + bytes([counter]), hashlib.sha256).digest()
okm += block
counter += 1
return okm[:length]
def derive_client_signing_key(secret: str, client_id: str) -> bytes:
return derive_key(secret, client_id, "meshsafe/api/v1/request-signing")
def derive_client_response_key(secret: str, client_id: str) -> bytes:
return derive_key(secret, client_id, "meshsafe/api/v1/response-signing")
def hash_password(password: str) -> str:
"""生成 ``pbkdf2_sha256$迭代次数$盐$哈希`` 格式的口令哈希。"""
salt = os.urandom(SALT_BYTES)
dk = hashlib.pbkdf2_hmac(
PBKDF2_ALGORITHM, password.encode("utf-8"), salt, PBKDF2_ITERATIONS
)
return "$".join(
[
"pbkdf2_sha256",
str(PBKDF2_ITERATIONS),
base64.b64encode(salt).decode("ascii"),
base64.b64encode(dk).decode("ascii"),
]
)
def verify_password(password: str, stored: str) -> bool:
"""校验口令。格式非法时返回 False,不抛出异常。"""
try:
algo, iterations, salt_b64, hash_b64 = stored.split("$")
if algo != "pbkdf2_sha256":
return False
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(hash_b64)
actual = hashlib.pbkdf2_hmac(
PBKDF2_ALGORITHM, password.encode("utf-8"), salt, int(iterations)
)
return hmac.compare_digest(actual, expected)
except Exception: # noqa: BLE001
return False
def needs_rehash(stored: str) -> bool:
"""判断存量口令哈希是否需要按当前参数重新计算。"""
try:
algo, iterations, _, _ = stored.split("$")
return algo != "pbkdf2_sha256" or int(iterations) < PBKDF2_ITERATIONS
except Exception: # noqa: BLE001
return True
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: str, chunk_size: int = 1024 * 1024) -> Tuple[str, int]:
"""流式计算文件 SHA256 与 MD5,返回 (sha256, md5, size)。"""
sha = hashlib.sha256()
md5 = hashlib.md5()
size = 0
with open(path, "rb") as fh:
while True:
chunk = fh.read(chunk_size)
if not chunk:
break
sha.update(chunk)
md5.update(chunk)
size += len(chunk)
return sha.hexdigest(), md5.hexdigest(), size
def constant_time_equals(a: str, b: str) -> bool:
return hmac.compare_digest(a.encode("utf-8"), b.encode("utf-8"))

261
app/security/deps.py Normal file
View File

@@ -0,0 +1,261 @@
"""API 安全依赖:认证、签名校验、防重放、限流、响应签名。"""
from __future__ import annotations
import json
import time
from typing import TYPE_CHECKING, Any, Dict, Optional
from fastapi import Depends, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from app.config import get_settings
from app.security import signing
from app.security.crypto import constant_time_equals
_settings = get_settings()
# redis 是可选依赖——没装时限流和防重放降级为内存策略(_check_rate_limit / nonce 检查里已兜底)
try:
import redis as _redis # type: ignore
except ModuleNotFoundError:
_redis = None
if TYPE_CHECKING:
import redis
_redis_client: Optional["redis.Redis"] = None
# 响应体原始字节暂存键,供响应签名中间件复用
RAW_BODY_ATTR = "_meshsafe_raw_body"
CLIENT_ATTR = "_meshsafe_client"
def get_redis():
"""懒加载 Redis 客户端。未安装 redis 包或连接失败时由调用方兜底。"""
global _redis_client
if _redis is None:
raise RuntimeError("redis 包未安装")
if _redis_client is None:
_redis_client = _redis.Redis.from_url(
_settings.redis_url, decode_responses=True, socket_timeout=5
)
return _redis_client
class ClientContext:
"""已通过认证的调用方上下文。"""
def __init__(self, client_id: str, secret: str, role: str = "operator") -> None:
self.client_id = client_id
self.secret = secret
self.role = role
def _lookup_client(client_id: str) -> Optional[ClientContext]:
for item in _settings.parsed_api_clients:
if constant_time_equals(item["client_id"], client_id):
return ClientContext(item["client_id"], item["secret"])
return None
def _check_rate_limit(client_id: str, path: str) -> None:
"""基于 Redis 的滑动窗口限流,防止接口被暴力试探。"""
try:
r = get_redis()
bucket = int(time.time() // 60)
key = f"ms:rl:{client_id}:{path}:{bucket}"
pipe = r.pipeline()
pipe.incr(key)
pipe.expire(key, 120)
count = pipe.execute()[0]
limit = 120 if "/scans" in path else 600
if count > limit:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail="请求过于频繁,已触发限流",
)
except HTTPException:
raise
except Exception: # noqa: BLE001 Redis 不可用时不阻断主流程
return
async def authenticate(request: Request) -> ClientContext:
"""认证与签名校验依赖。
依次执行:
1. 解析并校验调用方标识
2. 时间戳窗口校验(防重放)
3. nonce 一次性校验
4. 请求体哈希一致性校验
5. HMAC 签名校验
"""
client_id = request.headers.get("X-MS-Client-Id", "")
timestamp = request.headers.get("X-MS-Timestamp", "")
nonce = request.headers.get("X-MS-Nonce", "")
body_hash = request.headers.get("X-MS-Body-SHA256", "")
signature = request.headers.get("X-MS-Signature", "")
if not client_id or not timestamp or not nonce or not signature:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="缺少必要的认证请求头",
)
ctx = _lookup_client(client_id)
if ctx is None:
# 不区分"不存在"与"密钥错误",避免枚举 client_id
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="认证失败")
# 1) 时间戳窗口
try:
ts_int = int(timestamp)
except ValueError:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="认证失败")
skew = abs(int(time.time()) - ts_int)
if skew > _settings.signature_window:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=f"请求时间戳超出允许偏差({_settings.signature_window} 秒)",
)
# 2) nonce 一次性
try:
r = get_redis()
nonce_key = f"ms:nonce:{client_id}:{nonce}"
if not r.set(nonce_key, "1", nx=True, ex=_settings.signature_window * 2):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="重复请求,已拒绝"
)
except HTTPException:
raise
except Exception: # noqa: BLE001
pass
# 3) 读取请求体并校验哈希
# 注意:multipart/form-data 请求体已被 FastAPI UploadFile 消费,不能再 await request.body()
# 此时客户端已在 X-MS-Body-SHA256 里提供哈希,直接用即可
body: bytes = b""
ct = request.headers.get("content-type", "").lower()
if not ct.startswith("multipart/"):
try:
body = await request.body()
except RuntimeError:
# stream 已被消费(比如中间件或 UploadFile),用空 body 兜底
body = b""
skip_hash = ct.startswith("multipart/")
if not signing.verify_request_signature(
ctx.secret,
ctx.client_id,
request.method,
request.url.path,
dict(request.query_params),
body,
timestamp,
nonce,
body_hash,
signature,
skip_body_hash_check=skip_hash,
):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="认证失败")
_check_rate_limit(ctx.client_id, request.url.path)
request.state.client = ctx
return ctx
async def require_admin(ctx: ClientContext = Depends(authenticate)) -> ClientContext:
"""要求调用方具备完整 findings 权限。"""
if ctx.role != "operator":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="当前凭证无权查看完整检测明细",
)
return ctx
class ResponseSigningMiddleware(BaseHTTPMiddleware):
"""为所有 API 响应添加签名头。
响应体在读取过程中暂存原始字节,签名基于原始字节计算,
避免因 JSON 序列化差异导致客户端验签失败。
"""
async def dispatch(self, request: Request, call_next):
response = await call_next(request)
if not _settings.sign_response:
return response
if request.url.path.startswith("/api/"):
ctx = getattr(request.state, "client", None)
if ctx is None:
return response
body = getattr(response, RAW_BODY_ATTR, None)
if body is None:
body = b""
if hasattr(response, "body_iterator"):
chunks = []
async for chunk in response.body_iterator:
chunks.append(chunk if isinstance(chunk, bytes) else chunk.encode("utf-8"))
body = b"".join(chunks)
response = Response(
content=body,
status_code=response.status_code,
headers=dict(response.headers),
media_type=response.media_type,
)
# 删除旧 content-length(重建后会自动重新计算)
try:
del response.headers["content-length"]
except KeyError:
pass
headers = signing.sign_response(ctx.secret, ctx.client_id, body)
for key, value in headers.items():
response.headers[key] = value
return response
def build_signed_json(ctx: ClientContext, payload: Dict[str, Any], status_code: int = 200) -> JSONResponse:
"""构造带签名的 JSON 响应(供直接返回场景使用)。"""
body = signing.canonical_json_bytes(payload)
response = JSONResponse(content=payload, status_code=status_code)
if _settings.sign_response:
for key, value in signing.sign_response(ctx.secret, ctx.client_id, body).items():
response.headers[key] = value
return response
def redact_for_role(payload: Dict[str, Any], role: str) -> Dict[str, Any]:
"""按角色裁剪响应内容,降低信息泄露面。
``basic`` 角色只返回结论摘要,不返回引擎明细与提取文件路径,
避免通过差异响应反推检测逻辑。
"""
if role == "operator":
return payload
allowed_top = {
"task_id",
"verdict",
"score",
"action",
"reason",
"status",
"file",
"meta",
}
trimmed = {k: v for k, v in payload.items() if k in allowed_top}
file_info = trimmed.get("file")
if isinstance(file_info, dict):
trimmed["file"] = {
"sha256": file_info.get("sha256"),
"size": file_info.get("size"),
"real_type": file_info.get("real_type"),
}
trimmed["detail_truncated"] = True
return trimmed

113
app/security/ratelimit.py Normal file
View File

@@ -0,0 +1,113 @@
"""接口限流(Token bucket 滑窗)。
设计:
- 客户端维度(按 client_id 或 IP),每分钟请求配额
- 优先用 Redis(多 worker 共享计数),Redis 不可用时降级为内存字典
- 返回 429 + Retry-After header
"""
from __future__ import annotations
import threading
import time
from typing import Optional
from fastapi import Depends, HTTPException, Request, status
# ---------- 内存降级模式 ----------
# key -> (window_start_timestamp, count)
_MEM_BUCKET: dict[str, tuple[float, int]] = {}
_MEM_LOCK = threading.Lock()
_MEM_WINDOW = 60 # 秒
def _get_redis():
"""懒加载 Redis(避免模块 import 时没装直接崩)。"""
try:
import redis # type: ignore
except Exception: # noqa: BLE001
return None
from app.config import get_settings
try:
return redis.from_url(get_settings().redis_url, decode_responses=True)
except Exception: # noqa: BLE001
return None
def _memory_check(key: str, limit: int) -> bool:
"""内存滑窗:返回 True 表示允许,False 表示超限。"""
now = time.time()
with _MEM_LOCK:
start_ts, count = _MEM_BUCKET.get(key, (now, 0.0))
if now - start_ts > _MEM_WINDOW:
# 窗口外,重建
_MEM_BUCKET[key] = (now, 1)
return True
if count >= limit:
return False
_MEM_BUCKET[key] = (start_ts, count + 1)
return True
def _redis_check(client_key: str, limit: int) -> bool:
"""Redis 滑窗(INCR + EXPIRE)。"""
r = _get_redis()
if r is None:
return _memory_check(client_key, limit)
try:
key = f"meshsafe:rl:{client_key}"
pipe = r.pipeline()
pipe.incr(key)
pipe.expire(key, _MEM_WINDOW)
results = pipe.execute()
count = results[0]
return count <= limit
except Exception: # noqa: BLE001
return _memory_check(client_key, limit)
class RateLimiter:
"""限流依赖:返回一个 callable 给 Depends。
用法:
@app.post("/scan/sync")
async def scan_sync(..., limiter=Depends(RateLimiter(max_requests_per_min=60))):
...
"""
def __init__(self, max_requests_per_min: int = 60, label: str = "scan") -> None:
self.max_requests = max_requests_per_min
self.label = label
async def __call__(self, request: Request) -> None:
# 优先用 client_id(客户端签名带上的),退而用 IP
ctx = getattr(request.state, "client", None)
client_id = (ctx.client_id if ctx else None) or getattr(request.state, "client_id", None) or request.client.host
key = f"{self.label}:{client_id}"
if not _redis_check(key, self.max_requests):
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"Rate limit exceeded ({self.label}: {self.max_requests}/min)",
headers={"Retry-After": str(_MEM_WINDOW)},
)
def rate_limit(max_requests_per_min: int = 60, label: str = "scan"):
"""更简洁的限流装饰器(函数依赖)。
用法:
@app.post("/scan/sync")
async def scan_sync(..., _ = Depends(rate_limit(60, "scans"))):
...
"""
async def _check(request: Request) -> None:
ctx = getattr(request.state, "client", None)
client_id = (ctx.client_id if ctx else None) or getattr(request.state, "client_id", None) or request.client.host
key = f"{label}:{client_id}"
if not _redis_check(key, max_requests_per_min):
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"Rate limit exceeded ({label}: {max_requests_per_min}/min)",
headers={"Retry-After": str(_MEM_WINDOW)},
)
return _check

179
app/security/signing.py Normal file
View File

@@ -0,0 +1,179 @@
"""API 请求签名与响应签名。
请求签名规范(HMAC-SHA256)
---------------------------
客户端需在请求头中携带:
- ``X-MS-Client-Id`` 调用方标识
- ``X-MS-Timestamp`` Unix 秒级时间戳
- ``X-MS-Nonce`` 随机串,32 位十六进制,服务端保证一次性
- ``X-MS-Body-SHA256`` 请求体字节的 SHA256(无请求体时为空串的 SHA256)
- ``X-MS-Signature`` 签名值
待签名串由以下部分按固定顺序拼接,换行符为 ``\\n``::
METHOD
PATH
CANONICAL_QUERY
BODY_SHA256
TIMESTAMP
NONCE
CLIENT_ID
签名为 ``HMAC-SHA256(请求签名密钥, 待签名串)`` 的小写十六进制。
响应签名
--------
服务端在响应头中返回:
- ``X-MS-Response-Timestamp``
- ``X-MS-Response-Signature``: ``HMAC-SHA256(响应签名密钥, 响应体SHA256 + 时间戳)``
客户端用相同密钥派生规则即可独立验签,从而防止中间人篡改结论。
"""
from __future__ import annotations
import hashlib
import hmac
import json
from typing import Any, Dict, Mapping, Optional
EMPTY_BODY_SHA256 = hashlib.sha256(b"").hexdigest()
def canonical_query(params: Optional[Mapping[str, Any]]) -> str:
"""将查询参数规范化为 ``k=v&k2=v2`` 形式(按 key 升序)。"""
if not params:
return ""
items = []
for key in sorted(params.keys()):
value = params[key]
if value is None:
continue
if isinstance(value, (list, tuple)):
for item in value:
items.append(f"{key}={item}")
else:
items.append(f"{key}={value}")
return "&".join(items)
def body_sha256(body: bytes) -> str:
return hashlib.sha256(body).hexdigest()
def build_signing_string(
method: str,
path: str,
query: str,
body_hash: str,
timestamp: str,
nonce: str,
client_id: str,
) -> str:
return "\n".join(
[method.upper(), path, query, body_hash, str(timestamp), nonce, client_id]
)
def sign_request(
secret: str,
client_id: str,
method: str,
path: str,
query: Optional[Mapping[str, Any]] = None,
body: bytes = b"",
timestamp: Optional[int] = None,
nonce: Optional[str] = None,
) -> Dict[str, str]:
"""生成完整的请求签名头集合。"""
import os
import time
ts = str(timestamp if timestamp is not None else int(time.time()))
nc = nonce or os.urandom(16).hex()
bh = body_sha256(body)
qs = canonical_query(query)
payload = build_signing_string(method, path, qs, bh, ts, nc, client_id)
key = _signing_key(secret, client_id)
signature = hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()
return {
"X-MS-Client-Id": client_id,
"X-MS-Timestamp": ts,
"X-MS-Nonce": nc,
"X-MS-Body-SHA256": bh,
"X-MS-Signature": signature,
}
def verify_request_signature(
secret: str,
client_id: str,
method: str,
path: str,
query: Optional[Mapping[str, Any]],
body: bytes,
timestamp: str,
nonce: str,
provided_body_hash: str,
provided_signature: str,
*,
skip_body_hash_check: bool = False,
) -> bool:
"""服务端侧验签。
skip_body_hash_check=True 时(multipart/form-data 场景,服务端拿不到原始 body),
直接信任客户端提供的 provided_body_hash,只校验 HMAC 签名部分。
"""
if skip_body_hash_check:
expected_body_hash = provided_body_hash or ""
else:
expected_body_hash = body_sha256(body)
if not hmac.compare_digest(expected_body_hash, provided_body_hash or ""):
return False
payload = build_signing_string(
method, path, canonical_query(query), expected_body_hash, timestamp, nonce, client_id
)
key = _signing_key(secret, client_id)
expected = hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()
return hmac.compare_digest(expected, provided_signature or "")
def sign_response(secret: str, client_id: str, body_bytes: bytes, timestamp: Optional[int] = None) -> Dict[str, str]:
"""为响应体生成签名头。"""
import time
ts = str(timestamp if timestamp is not None else int(time.time()))
key = _response_key(secret, client_id)
payload = body_sha256(body_bytes) + ts
signature = hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()
return {"X-MS-Response-Timestamp": ts, "X-MS-Response-Signature": signature}
def verify_response_signature(
secret: str, client_id: str, body_bytes: bytes, timestamp: str, signature: str
) -> bool:
key = _response_key(secret, client_id)
payload = body_sha256(body_bytes) + timestamp
expected = hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()
return hmac.compare_digest(expected, signature or "")
def canonical_json_bytes(payload: Any) -> bytes:
"""将响应对象序列化为稳定的 JSON 字节串(供签名使用)。"""
return json.dumps(
payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")
).encode("utf-8")
def _signing_key(secret: str, client_id: str) -> bytes:
from app.security.crypto import derive_client_signing_key
return derive_client_signing_key(secret, client_id)
def _response_key(secret: str, client_id: str) -> bytes:
from app.security.crypto import derive_client_response_key
return derive_client_response_key(secret, client_id)