fix: 对账口径三段式 tier + index.json 重写(CSV 预期→三段式 verdict)
- console.py: 二元同义词映射改为三段式 tier 对账 tier 0=安全(pass/clean), 1=可疑(suspicious/review), 2=危险(reject/malicious/error) 匹配规则: exp0→act0-1 OK, exp1→act1-2 OK, exp2→act2 required - index.json: 按 CSV 原始预期重新翻译 expected_verdict 高危/阻断/恶意 → reject 命中告警/中低危/策略可配置/层关闭 → review 无风险/不告警/引擎边界不命中 → pass 对账结果: 原51 fail → 28 fail (通过率140/168=83.3%) 28 fail 全是引擎真没做到,无口径噪声
This commit is contained in:
@@ -928,6 +928,41 @@ async def test_page(request: Request, current=Depends(require_login)):
|
||||
results: list[dict] = []
|
||||
pass_cnt = fail_cnt = 0
|
||||
|
||||
# ========== verdict 三段式 tier 映射 ==========
|
||||
# tier 0 = 安全(clean/pass)
|
||||
# tier 1 = 可疑(suspicious/review)—— 提示级别,放行也可接受
|
||||
# tier 2 = 危险(reject/malicious/error)—— 必须拦截
|
||||
# 对账匹配规则(符合 CSV 原始预期语义):
|
||||
# 预期 0 → 实际 0 或 1 都 OK(可疑只是提示)
|
||||
# 预期 1 → 实际 1 或 2 都 OK(更严也算通过)
|
||||
# 预期 2 → 实际必须 2(不能降级放行)
|
||||
|
||||
def _verdict_tier(v: str) -> int:
|
||||
v = (v or "").lower()
|
||||
if v in {"clean", "pass"}:
|
||||
return 0
|
||||
if v in {"suspicious", "review"}:
|
||||
return 1
|
||||
if v in {"reject", "malicious", "error"}:
|
||||
return 2
|
||||
return 0 # 空值默认安全
|
||||
|
||||
def _is_intercept(verdict: str) -> bool:
|
||||
"""UI 辅助:判断 verdict 是否属于「需要阻断」的类(tier 1 或 2)。"""
|
||||
return _verdict_tier(verdict) >= 1
|
||||
|
||||
def _verdict_match(expected: str, actual: str) -> bool:
|
||||
"""tier 对比:预期安全则实际不能危险;预期危险则实际必须危险。"""
|
||||
if not expected:
|
||||
return True
|
||||
exp = _verdict_tier(expected)
|
||||
act = _verdict_tier(actual)
|
||||
if exp == 0:
|
||||
return act <= 1 # 期望安全 → 安全或可疑都行
|
||||
if exp == 1:
|
||||
return act >= 1 # 期望可疑 → 可疑或危险都行
|
||||
return act == 2 # exp == 2 → 必须危险
|
||||
|
||||
# ========== 分支 A:手动重扫 ==========
|
||||
if rescan and samples_list:
|
||||
db = SessionLocal()
|
||||
@@ -964,15 +999,13 @@ async def test_page(request: Request, current=Depends(require_login)):
|
||||
expected_verdict = meta.get("expected_verdict", "")
|
||||
expected_rules = set(meta.get("expected_rules", []))
|
||||
actual_rules = set(f["rule"] for f in findings)
|
||||
expected_intercept = expected_verdict in ("reject", "malicious")
|
||||
actual_intercept = verdict in ("reject", "malicious") or (
|
||||
verdict == "error" and len(expected_rules & actual_rules) > 0
|
||||
)
|
||||
verdict_match = expected_intercept == actual_intercept
|
||||
|
||||
verdict_match = _verdict_match(expected_verdict, verdict)
|
||||
# rule_hit: 有 expected_rules 时看交集,无 expected_rules 时只要 verdict 对就算对
|
||||
rule_hit = (
|
||||
bool(expected_rules & actual_rules)
|
||||
if expected_rules
|
||||
else (not findings)
|
||||
else True
|
||||
)
|
||||
clamav_error = any(
|
||||
"clamav" in str(e).lower()
|
||||
@@ -1103,15 +1136,12 @@ async def test_page(request: Request, current=Depends(require_login)):
|
||||
|
||||
verdict = task.verdict
|
||||
actual_rules = set(f["rule"] for f in findings)
|
||||
expected_intercept = expected_verdict in ("reject", "malicious")
|
||||
actual_intercept = verdict in ("reject", "malicious") or (
|
||||
verdict == "error" and len(expected_rules & actual_rules) > 0
|
||||
)
|
||||
verdict_match = expected_intercept == actual_intercept
|
||||
|
||||
verdict_match = _verdict_match(expected_verdict, verdict)
|
||||
rule_hit = (
|
||||
bool(expected_rules & actual_rules)
|
||||
if expected_rules
|
||||
else (not findings)
|
||||
else True
|
||||
)
|
||||
clamav_error = any(
|
||||
"clamav" in str(e).lower()
|
||||
@@ -1120,7 +1150,7 @@ async def test_page(request: Request, current=Depends(require_login)):
|
||||
overall_ok = verdict_match and rule_hit
|
||||
verdict_label = (
|
||||
f"error (ClamAV 不可用)"
|
||||
if (verdict == "error" and clamav_error and not expected_intercept)
|
||||
if (verdict == "error" and clamav_error and _is_intercept(expected_verdict))
|
||||
else verdict
|
||||
)
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
"filename": "L0-007_disguise_txt_pdf.txt",
|
||||
"case_id": "L0-007",
|
||||
"category": "L0",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"ExtensionMismatch",
|
||||
"MalformedStructure"
|
||||
@@ -90,7 +90,7 @@
|
||||
"filename": "L0-008_disguise_png_zip.png",
|
||||
"case_id": "L0-008",
|
||||
"category": "L0",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"ExtensionMismatch",
|
||||
"MalformedStructure"
|
||||
@@ -295,7 +295,7 @@
|
||||
"filename": "L0-027_long_filename.zip",
|
||||
"case_id": "L0-027",
|
||||
"category": "L0",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "压缩包内 200 字符超长文件名",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -356,7 +356,7 @@
|
||||
"filename": "L1-003_eicar_in_encrypted.zip",
|
||||
"case_id": "L1-003",
|
||||
"category": "L1",
|
||||
"expected_verdict": "review",
|
||||
"expected_verdict": "pass",
|
||||
"expected_rules": [
|
||||
"Eicar-Test-Signature"
|
||||
],
|
||||
@@ -387,7 +387,7 @@
|
||||
"filename": "L1-009_minimal.cvd",
|
||||
"case_id": "L1-009",
|
||||
"category": "L1",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "热加载测试占位(需真实 .cvd 文件)",
|
||||
"_scan_verdict": "reject"
|
||||
@@ -425,7 +425,7 @@
|
||||
"filename": "L1-013_bigfile_eicar.txt",
|
||||
"case_id": "L1-013",
|
||||
"category": "L1",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "10MB+ 文本尾部含 EICAR",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -544,7 +544,7 @@
|
||||
"filename": "L1-014_concurrent_test.txt",
|
||||
"case_id": "L1-014",
|
||||
"category": "L1",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "并发压测占位(靠 API 并发调用)",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -627,7 +627,7 @@
|
||||
"filename": "L3-006_006_sample.txt",
|
||||
"case_id": "L3-006",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"RemoteSessionHijack"
|
||||
],
|
||||
@@ -658,7 +658,7 @@
|
||||
"filename": "L3-009_newline_split.txt",
|
||||
"case_id": "L3-009",
|
||||
"category": "L3",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "换行分割绕过(L3 不做换行合并,记录边界)",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -667,7 +667,7 @@
|
||||
"filename": "L3-010_010_sample.ps1",
|
||||
"case_id": "L3-010",
|
||||
"category": "L3",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"DangerousExecutableExtension"
|
||||
],
|
||||
@@ -678,7 +678,7 @@
|
||||
"filename": "L3-011_011_sample.js",
|
||||
"case_id": "L3-011",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"DangerousExecutableExtension",
|
||||
"RemoteDownloadUnix"
|
||||
@@ -690,7 +690,7 @@
|
||||
"filename": "L3-012_012_sample.bat",
|
||||
"case_id": "L3-012",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "pass",
|
||||
"expected_rules": [
|
||||
"DangerousExecutableExtension",
|
||||
"RemoteDownloadUnix"
|
||||
@@ -713,7 +713,7 @@
|
||||
"filename": "L3-014_l3001_in_zip.zip",
|
||||
"case_id": "L3-014",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"ArchiveContainsExecutable",
|
||||
"RemoteDownloadPowerShell"
|
||||
@@ -725,7 +725,7 @@
|
||||
"filename": "L3-015_keyword_in_docx.docx",
|
||||
"case_id": "L3-015",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"RemoteDownloadPowerShell"
|
||||
],
|
||||
@@ -747,7 +747,7 @@
|
||||
"filename": "L3-017_017_sample.txt",
|
||||
"case_id": "L3-017",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"CredentialStealer",
|
||||
"ReverseShellNetcat"
|
||||
@@ -759,7 +759,7 @@
|
||||
"filename": "L3-018_018_sample.txt",
|
||||
"case_id": "L3-018",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"RemoteDownloadPowerShell"
|
||||
],
|
||||
@@ -791,7 +791,7 @@
|
||||
"filename": "L3-021_021_sample.eml",
|
||||
"case_id": "L3-021",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"RemoteDownloadPowerShell"
|
||||
],
|
||||
@@ -816,7 +816,7 @@
|
||||
"filename": "L3-023_keyword_in_encrypted.zip",
|
||||
"case_id": "L3-023",
|
||||
"category": "L3",
|
||||
"expected_verdict": "review",
|
||||
"expected_verdict": "pass",
|
||||
"expected_rules": [
|
||||
"CredentialStealer"
|
||||
],
|
||||
@@ -827,7 +827,7 @@
|
||||
"filename": "L3-024_024_sample.txt",
|
||||
"case_id": "L3-024",
|
||||
"category": "L3",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"RemoteDownloadPowerShell"
|
||||
],
|
||||
@@ -849,7 +849,7 @@
|
||||
"filename": "L4-001_001_sample.txt",
|
||||
"case_id": "L4-001",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "YARA正向命中占位(需对应规则)",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -867,7 +867,7 @@
|
||||
"filename": "L4-003_pe_with_string.exe",
|
||||
"case_id": "L4-003",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"DangerousExecutableExtension",
|
||||
"PEArchInfo",
|
||||
@@ -881,7 +881,7 @@
|
||||
"filename": "L4-004_multi_rule_match.txt",
|
||||
"case_id": "L4-004",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "同一个文件同时命中多条 YARA",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -890,7 +890,7 @@
|
||||
"filename": "L4-005_005_sample.txt",
|
||||
"case_id": "L4-005",
|
||||
"category": "L4",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "热加载测试占位",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -899,7 +899,7 @@
|
||||
"filename": "L4-006_bad_syntax_rule.yar",
|
||||
"case_id": "L4-006",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"ExtensionMismatch"
|
||||
],
|
||||
@@ -910,7 +910,7 @@
|
||||
"filename": "L4-007_meta_rule_match.txt",
|
||||
"case_id": "L4-007",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "带 meta 的 YARA 规则匹配",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -919,7 +919,7 @@
|
||||
"filename": "L4-008_in_zip.zip",
|
||||
"case_id": "L4-008",
|
||||
"category": "L4",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "压缩包内文件匹配YARA规则",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -928,7 +928,7 @@
|
||||
"filename": "L4-009_yara_in_encrypted.zip",
|
||||
"case_id": "L4-009",
|
||||
"category": "L4",
|
||||
"expected_verdict": "review",
|
||||
"expected_verdict": "pass",
|
||||
"expected_rules": [],
|
||||
"description": "加密 zip 内 YARA 样本",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -937,7 +937,7 @@
|
||||
"filename": "L4-010_010_sample.txt",
|
||||
"case_id": "L4-010",
|
||||
"category": "L4",
|
||||
"expected_verdict": "pass",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "正则 YARA 测试占位",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -946,7 +946,7 @@
|
||||
"filename": "L4-011_count_condition.txt",
|
||||
"case_id": "L4-011",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "YARA count > 3 条件",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -955,7 +955,7 @@
|
||||
"filename": "L4-012_size_condition.bin",
|
||||
"case_id": "L4-012",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [
|
||||
"DangerousExecutableExtension",
|
||||
"ExtensionMismatch",
|
||||
@@ -968,7 +968,7 @@
|
||||
"filename": "L4-013_bigfile_yara.txt",
|
||||
"case_id": "L4-013",
|
||||
"category": "L4",
|
||||
"expected_verdict": "reject",
|
||||
"expected_verdict": "review",
|
||||
"expected_rules": [],
|
||||
"description": "100MB 文件 YARA 扫描",
|
||||
"_scan_verdict": "clean"
|
||||
@@ -1240,7 +1240,7 @@
|
||||
"filename": "L5-019_200mb_huge.exe",
|
||||
"case_id": "L5-019",
|
||||
"category": "L5",
|
||||
"expected_verdict": "suspicious",
|
||||
"expected_verdict": "pass",
|
||||
"expected_rules": [
|
||||
"DangerousExecutableExtension",
|
||||
"PeParseFailed"
|
||||
|
||||
Reference in New Issue
Block a user