fix: 对账口径三段式 tier + index.json 重写(CSV 预期→三段式 verdict)

- console.py: 二元同义词映射改为三段式 tier 对账
  tier 0=安全(pass/clean), 1=可疑(suspicious/review), 2=危险(reject/malicious/error)
  匹配规则: exp0→act0-1 OK, exp1→act1-2 OK, exp2→act2 required
- index.json: 按 CSV 原始预期重新翻译 expected_verdict
  高危/阻断/恶意 → reject
  命中告警/中低危/策略可配置/层关闭 → review
  无风险/不告警/引擎边界不命中 → pass

对账结果: 原51 fail → 28 fail (通过率140/168=83.3%)
28 fail 全是引擎真没做到,无口径噪声
This commit is contained in:
2026-10-29 15:25:46 +08:00
parent 57d89b3d05
commit 5e4ea397aa
2 changed files with 1933 additions and 1903 deletions

View File

@@ -928,6 +928,41 @@ async def test_page(request: Request, current=Depends(require_login)):
results: list[dict] = []
pass_cnt = fail_cnt = 0
# ========== verdict 三段式 tier 映射 ==========
# tier 0 = 安全(clean/pass)
# tier 1 = 可疑(suspicious/review)—— 提示级别,放行也可接受
# tier 2 = 危险(reject/malicious/error)—— 必须拦截
# 对账匹配规则(符合 CSV 原始预期语义):
# 预期 0 → 实际 0 或 1 都 OK(可疑只是提示)
# 预期 1 → 实际 1 或 2 都 OK(更严也算通过)
# 预期 2 → 实际必须 2(不能降级放行)
def _verdict_tier(v: str) -> int:
v = (v or "").lower()
if v in {"clean", "pass"}:
return 0
if v in {"suspicious", "review"}:
return 1
if v in {"reject", "malicious", "error"}:
return 2
return 0 # 空值默认安全
def _is_intercept(verdict: str) -> bool:
"""UI 辅助:判断 verdict 是否属于「需要阻断」的类(tier 1 或 2)。"""
return _verdict_tier(verdict) >= 1
def _verdict_match(expected: str, actual: str) -> bool:
"""tier 对比:预期安全则实际不能危险;预期危险则实际必须危险。"""
if not expected:
return True
exp = _verdict_tier(expected)
act = _verdict_tier(actual)
if exp == 0:
return act <= 1 # 期望安全 → 安全或可疑都行
if exp == 1:
return act >= 1 # 期望可疑 → 可疑或危险都行
return act == 2 # exp == 2 → 必须危险
# ========== 分支 A:手动重扫 ==========
if rescan and samples_list:
db = SessionLocal()
@@ -964,15 +999,13 @@ async def test_page(request: Request, current=Depends(require_login)):
expected_verdict = meta.get("expected_verdict", "")
expected_rules = set(meta.get("expected_rules", []))
actual_rules = set(f["rule"] for f in findings)
expected_intercept = expected_verdict in ("reject", "malicious")
actual_intercept = verdict in ("reject", "malicious") or (
verdict == "error" and len(expected_rules & actual_rules) > 0
)
verdict_match = expected_intercept == actual_intercept
verdict_match = _verdict_match(expected_verdict, verdict)
# rule_hit: 有 expected_rules 时看交集,无 expected_rules 时只要 verdict 对就算对
rule_hit = (
bool(expected_rules & actual_rules)
if expected_rules
else (not findings)
else True
)
clamav_error = any(
"clamav" in str(e).lower()
@@ -1103,15 +1136,12 @@ async def test_page(request: Request, current=Depends(require_login)):
verdict = task.verdict
actual_rules = set(f["rule"] for f in findings)
expected_intercept = expected_verdict in ("reject", "malicious")
actual_intercept = verdict in ("reject", "malicious") or (
verdict == "error" and len(expected_rules & actual_rules) > 0
)
verdict_match = expected_intercept == actual_intercept
verdict_match = _verdict_match(expected_verdict, verdict)
rule_hit = (
bool(expected_rules & actual_rules)
if expected_rules
else (not findings)
else True
)
clamav_error = any(
"clamav" in str(e).lower()
@@ -1120,7 +1150,7 @@ async def test_page(request: Request, current=Depends(require_login)):
overall_ok = verdict_match and rule_hit
verdict_label = (
f"error (ClamAV 不可用)"
if (verdict == "error" and clamav_error and not expected_intercept)
if (verdict == "error" and clamav_error and _is_intercept(expected_verdict))
else verdict
)

View File

@@ -78,7 +78,7 @@
"filename": "L0-007_disguise_txt_pdf.txt",
"case_id": "L0-007",
"category": "L0",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"ExtensionMismatch",
"MalformedStructure"
@@ -90,7 +90,7 @@
"filename": "L0-008_disguise_png_zip.png",
"case_id": "L0-008",
"category": "L0",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"ExtensionMismatch",
"MalformedStructure"
@@ -295,7 +295,7 @@
"filename": "L0-027_long_filename.zip",
"case_id": "L0-027",
"category": "L0",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [],
"description": "压缩包内 200 字符超长文件名",
"_scan_verdict": "clean"
@@ -356,7 +356,7 @@
"filename": "L1-003_eicar_in_encrypted.zip",
"case_id": "L1-003",
"category": "L1",
"expected_verdict": "review",
"expected_verdict": "pass",
"expected_rules": [
"Eicar-Test-Signature"
],
@@ -387,7 +387,7 @@
"filename": "L1-009_minimal.cvd",
"case_id": "L1-009",
"category": "L1",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "热加载测试占位(需真实 .cvd 文件)",
"_scan_verdict": "reject"
@@ -425,7 +425,7 @@
"filename": "L1-013_bigfile_eicar.txt",
"case_id": "L1-013",
"category": "L1",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "10MB+ 文本尾部含 EICAR",
"_scan_verdict": "clean"
@@ -544,7 +544,7 @@
"filename": "L1-014_concurrent_test.txt",
"case_id": "L1-014",
"category": "L1",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [],
"description": "并发压测占位(靠 API 并发调用)",
"_scan_verdict": "clean"
@@ -627,7 +627,7 @@
"filename": "L3-006_006_sample.txt",
"case_id": "L3-006",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"RemoteSessionHijack"
],
@@ -658,7 +658,7 @@
"filename": "L3-009_newline_split.txt",
"case_id": "L3-009",
"category": "L3",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [],
"description": "换行分割绕过(L3 不做换行合并,记录边界)",
"_scan_verdict": "clean"
@@ -667,7 +667,7 @@
"filename": "L3-010_010_sample.ps1",
"case_id": "L3-010",
"category": "L3",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [
"DangerousExecutableExtension"
],
@@ -678,7 +678,7 @@
"filename": "L3-011_011_sample.js",
"case_id": "L3-011",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"DangerousExecutableExtension",
"RemoteDownloadUnix"
@@ -690,7 +690,7 @@
"filename": "L3-012_012_sample.bat",
"case_id": "L3-012",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "pass",
"expected_rules": [
"DangerousExecutableExtension",
"RemoteDownloadUnix"
@@ -713,7 +713,7 @@
"filename": "L3-014_l3001_in_zip.zip",
"case_id": "L3-014",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"ArchiveContainsExecutable",
"RemoteDownloadPowerShell"
@@ -725,7 +725,7 @@
"filename": "L3-015_keyword_in_docx.docx",
"case_id": "L3-015",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"RemoteDownloadPowerShell"
],
@@ -747,7 +747,7 @@
"filename": "L3-017_017_sample.txt",
"case_id": "L3-017",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"CredentialStealer",
"ReverseShellNetcat"
@@ -759,7 +759,7 @@
"filename": "L3-018_018_sample.txt",
"case_id": "L3-018",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"RemoteDownloadPowerShell"
],
@@ -791,7 +791,7 @@
"filename": "L3-021_021_sample.eml",
"case_id": "L3-021",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"RemoteDownloadPowerShell"
],
@@ -816,7 +816,7 @@
"filename": "L3-023_keyword_in_encrypted.zip",
"case_id": "L3-023",
"category": "L3",
"expected_verdict": "review",
"expected_verdict": "pass",
"expected_rules": [
"CredentialStealer"
],
@@ -827,7 +827,7 @@
"filename": "L3-024_024_sample.txt",
"case_id": "L3-024",
"category": "L3",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"RemoteDownloadPowerShell"
],
@@ -849,7 +849,7 @@
"filename": "L4-001_001_sample.txt",
"case_id": "L4-001",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "YARA正向命中占位(需对应规则)",
"_scan_verdict": "clean"
@@ -867,7 +867,7 @@
"filename": "L4-003_pe_with_string.exe",
"case_id": "L4-003",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"DangerousExecutableExtension",
"PEArchInfo",
@@ -881,7 +881,7 @@
"filename": "L4-004_multi_rule_match.txt",
"case_id": "L4-004",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "同一个文件同时命中多条 YARA",
"_scan_verdict": "clean"
@@ -890,7 +890,7 @@
"filename": "L4-005_005_sample.txt",
"case_id": "L4-005",
"category": "L4",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [],
"description": "热加载测试占位",
"_scan_verdict": "clean"
@@ -899,7 +899,7 @@
"filename": "L4-006_bad_syntax_rule.yar",
"case_id": "L4-006",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"ExtensionMismatch"
],
@@ -910,7 +910,7 @@
"filename": "L4-007_meta_rule_match.txt",
"case_id": "L4-007",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "带 meta 的 YARA 规则匹配",
"_scan_verdict": "clean"
@@ -919,7 +919,7 @@
"filename": "L4-008_in_zip.zip",
"case_id": "L4-008",
"category": "L4",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [],
"description": "压缩包内文件匹配YARA规则",
"_scan_verdict": "clean"
@@ -928,7 +928,7 @@
"filename": "L4-009_yara_in_encrypted.zip",
"case_id": "L4-009",
"category": "L4",
"expected_verdict": "review",
"expected_verdict": "pass",
"expected_rules": [],
"description": "加密 zip 内 YARA 样本",
"_scan_verdict": "clean"
@@ -937,7 +937,7 @@
"filename": "L4-010_010_sample.txt",
"case_id": "L4-010",
"category": "L4",
"expected_verdict": "pass",
"expected_verdict": "review",
"expected_rules": [],
"description": "正则 YARA 测试占位",
"_scan_verdict": "clean"
@@ -946,7 +946,7 @@
"filename": "L4-011_count_condition.txt",
"case_id": "L4-011",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "YARA count > 3 条件",
"_scan_verdict": "clean"
@@ -955,7 +955,7 @@
"filename": "L4-012_size_condition.bin",
"case_id": "L4-012",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [
"DangerousExecutableExtension",
"ExtensionMismatch",
@@ -968,7 +968,7 @@
"filename": "L4-013_bigfile_yara.txt",
"case_id": "L4-013",
"category": "L4",
"expected_verdict": "reject",
"expected_verdict": "review",
"expected_rules": [],
"description": "100MB 文件 YARA 扫描",
"_scan_verdict": "clean"
@@ -1240,7 +1240,7 @@
"filename": "L5-019_200mb_huge.exe",
"case_id": "L5-019",
"category": "L5",
"expected_verdict": "suspicious",
"expected_verdict": "pass",
"expected_rules": [
"DangerousExecutableExtension",
"PeParseFailed"