From 57d89b3d05c4da0d890db506582c6445c84a9ffa Mon Sep 17 00:00:00 2001 From: zhaolei <12153398@qq.com> Date: Thu, 29 Oct 2026 14:40:49 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20A+B=20=E7=BC=BA=E5=8F=A3=E5=85=A8?= =?UTF-8?q?=E8=A1=A5=20+=20README=20=E5=BE=85=E5=8A=9E=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 结构层小缺口 (3/3): ✅ structure.py 加 LongFilename 告警 (>120 字符成员名) ✅ extractor.py _SAFE_NAME_RE 扩展 emoji 范围 (保留 emoji 文件名不崩) ✅ 重生成 L0-023_5layer_nested.zip (5 层真嵌套) B 白名单机制 (2/2): ✅ policy.yaml 加 keyword_whitelist (内网 IP/域名自动放行 RemoteDownloadUnix/PowerShell) ✅ policy.yaml 加 yara_hash_whitelist (MD5 在白名单跳过 YARA 告警) ✅ policy.py Property 类加两个 property 读取白名单 ✅ policy.py PolicyEngine.decide() 步骤 -1: 白名单过滤 (优先级最高) ✅ scanner.py decide() 调用补传 md5 外部资源缺口 (3 条，README 已记录): - L1-009/010 需真实签名 .cvd 文件 - L1-014 需压测脚本 stress_test.py 测试靶场: ✅ rules/test_rules.yar (7 条自动生成 YARA 规则) ✅ README 新增测试靶场待办 + 白名单机制说明 + 脚本清单 --- README.md | 26 +++++++++++++++ app/engines/extractor.py | 9 ++++- app/engines/structure.py | 15 +++++++++ app/services/policy.py | 44 +++++++++++++++++++++++++ app/services/scanner.py | 1 + config/policy.yaml | 31 +++++++++++++++++ tests/samples/L0-023_5layer_nested.zip | Bin 640 -> 952 bytes 7 files changed, 125 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 540ed97..f857137 100644 --- a/README.md +++ b/README.md @@ -224,4 +224,30 @@ MeshSafe 会**分析宏的实际行为**： --- +## 测试靶场（待办） + +CSV 用例集共 126 条，引擎能力覆盖约 97%。以下 3 条因需要外部资源暂搁置，待办环境就绪后补全： + +| CSV 用例 | 内容 | 缺失原因 | 后续 | +|----------|------|---------|------| +| **L1-009** | ClamAV 离线病毒库热加载 | 需要真实签名的 `.cvd` 文件（ClamAV 官方下载） | 从 https://www.clamav.net/downloads 下载 daily.cvd，放入 `data/clamav-db/` 测试 | +| **L1-010** | ClamAV 损坏库容错降级 | 需要损坏的真实 `.cvd`（自制损坏文件可能导致 clamd 无法启动） | 用 `sigtool --unpack` 解包后篡改一个字节，再重新打包测试 | +| **L1-014** | 并发多 EICAR 压测 | 不是引擎功能，需要压测框架（并发 httpx 提交 + 监控引擎内存/耗时） | 写 `scripts/stress_test.py`，用 asyncio.Semaphore 控制并发度 | + +### 白名单机制（v0.2.1 新增） + +- **关键词白名单** (`policy.yaml → keyword_whitelist`)：规则命中后若 detail 包含内网 IP/域名，自动放行 +- **YARA 哈希白名单** (`policy.yaml → yara_hash_whitelist`)：YARA 命中后 MD5 在白名单 → 跳过告警 +- 生产部署时按需补充内网地址段 / 已知安全工具哈希 + +### 靶场样本生成脚本 + +| 脚本 | 用途 | +|------|------| +| `scripts/generate_csv_samples.py` | 从 CSV 批量生成 CSV 对齐样本 + index.json | +| `scripts/generate_yara_rules.py` | 自动生成 CSV L4 段对应的 YARA `.yar` 规则 | +| `rules/test_rules.yar` | 生成的规则文件（引擎按 mtime 自动热加载） | + +--- + *更多技术细节请参阅 [总体方案](docs/00-总体方案.md) 和 [接口文档](docs/接口文档.md)* diff --git a/app/engines/extractor.py b/app/engines/extractor.py index c92ce06..cfacf3e 100644 --- a/app/engines/extractor.py +++ b/app/engines/extractor.py @@ -27,7 +27,14 @@ from app.engines.base import Finding, ScanTarget, Severity ARCHIVE_EXTS = {".zip", ".jar", ".war", ".apk", ".epub", ".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp"} TAR_EXTS = {".tar", ".gz", ".tgz", ".bz2", ".xz", ".tbz2", ".txz"} -_SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._一-鿿-]+") +_SAFE_NAME_RE = re.compile( + r"[^A-Za-z0-9._一-鿿" # ASCII + CJK 统一汉字 + r"\U0001F300-\U0001FAFF" # Emoji 表情符号 + r"\U0001F000-\U0001F2FF" # 符号和象形文字扩展 A + r"\U00002600-\U000027BF" # 杂项符号 + 装饰符号 + r"\U00002B00-\U00002BFF" # 箭头等 + r"-]+" +) @dataclass diff --git a/app/engines/structure.py b/app/engines/structure.py index d86516c..06c2ad4 100644 --- a/app/engines/structure.py +++ b/app/engines/structure.py @@ -869,6 +869,7 @@ class StructureEngine(Engine): total_uncompressed = 0 executable_members: List[str] = [] traversal_members: List[str] = [] + long_filename_members: List[str] = [] member_count = 0 encrypted_count = 0 @@ -886,6 +887,8 @@ class StructureEngine(Engine): traversal_members.append(name) if info.flag_bits & 0x1: # ZIP 加密标志（bit 0） encrypted_count += 1 + if len(os.path.basename(name)) > 120: + long_filename_members.append(name) except Exception as exc: # noqa: BLE001 return [ Finding( @@ -970,6 +973,18 @@ class StructureEngine(Engine): ) ) + if long_filename_members: + findings.append( + Finding( + engine=self.name, + layer=self.layer, + severity=Severity.LOW.value, + rule="LongFilename", + detail=f"归档内存在超长文件名（>120 字符）{len(long_filename_members)} 个，可能规避扫描或探测边界", + weight=5.0, + ) + ) + return findings def _inspect_tar(self, target: ScanTarget, max_ratio: int, max_total: int) -> List[Finding]: diff --git a/app/services/policy.py b/app/services/policy.py index 13ae01e..4f587d4 100644 --- a/app/services/policy.py +++ b/app/services/policy.py @@ -99,6 +99,17 @@ class Policy: """内网特征词表规则（L2B 引擎使用）。""" return self.raw.get("keyword_rules", []) or [] + @property + def keyword_whitelist(self) -> List[Dict[str, Any]]: + """关键词白名单。规则命中后，若 detail 包含 allowed_keywords 中任一子串 → 过滤。""" + return self.raw.get("keyword_whitelist", []) or [] + + @property + def yara_hash_whitelist_set(self) -> set[str]: + """YARA 哈希白名单（返回 MD5 集合，便于快速查询）。""" + raw = self.raw.get("yara_hash_whitelist", []) or [] + return {str(entry.get("md5", "")).lower() for entry in raw if isinstance(entry, dict) and entry.get("md5")} + @property def yara_weights(self) -> Dict[str, float]: """YARA 规则权重映射：规则名 → weight(0-100)。 @@ -387,6 +398,7 @@ class PolicyEngine: real_type: str = "", engine_errors: Optional[List[str]] = None, encrypted: bool = False, + md5: str = "", ) -> Decision: """综合所有信号产出最终裁决。 @@ -410,6 +422,38 @@ class PolicyEngine: ext = (declared_ext or "").lower() + # ====================================================================== + # 步骤 -1：白名单过滤（优先级最高） + # ====================================================================== + # 命中白名单的规则从 findings 中移除，不再参与任何后续裁决。 + # 包括：关键词白名单（按 rule + detail 子串匹配）+ YARA 哈希白名单。 + # ====================================================================== + wl_filtered: List[Finding] = [] + keyword_wl = policy.keyword_whitelist + for f in findings: + skip = False + # 关键词白名单：detail 包含 allowed_keywords 任一子串 + if keyword_wl and f.engine == "keywords": + for wl_entry in keyword_wl: + if f.rule == wl_entry.get("rule"): + allowed = wl_entry.get("allowed_keywords", []) + if allowed and any(kw in f.detail for kw in allowed): + skip = True + logger.info( + "白名单放行：rule=%s 命中关键词白名单 detail=%s", + f.rule, f.detail, + ) + break + # YARA 哈希白名单：md5 在白名单集合中 + if not skip and f.engine == "yara": + wl_set = policy.yara_hash_whitelist_set + if wl_set and md5 and md5.lower() in wl_set: + skip = True + logger.info("白名单放行：YARA 命中但 MD5 在哈希白名单") + if not skip: + wl_filtered.append(f) + findings = wl_filtered + # ====================================================================== # 步骤 0：零容忍规则优先检查 # ====================================================================== diff --git a/app/services/scanner.py b/app/services/scanner.py index 1a6dfd9..a79b819 100644 --- a/app/services/scanner.py +++ b/app/services/scanner.py @@ -453,6 +453,7 @@ class Scanner: real_type=real_type, engine_errors=engine_errors, encrypted=encrypted, + md5=root_info.get("md5", ""), ) # 计算 open_risk（独立于裁决结果，供人阅读） diff --git a/config/policy.yaml b/config/policy.yaml index 2adfe73..a1e418a 100644 --- a/config/policy.yaml +++ b/config/policy.yaml @@ -398,6 +398,37 @@ keyword_rules: weight: 30 severity: high +# ---------- 白名单（规则命中后二次放宽） ---------- +# 优先级：白名单 > 关键词规则 > YARA 规则 > 评分累加 +# 命中白名单的规则会从 findings 中过滤，不再进入裁决 + +keyword_whitelist: + # ---- 内网合法运维地址/域名（白名单放行 curl/wget 下载）---- + - rule: RemoteDownloadUnix # curl/wget 下载特征 + allowed_keywords: + - "192.168." + - "10." + - "172.16." + - "172.17." + - "172.18." + - "172.19." + - "172.2" + - "172.3" + - ".senmesh.com" + - "repo.internal" + - "mirror.internal" + - rule: RemoteDownloadPowerShell # Invoke-WebRequest/BitsAdmin + allowed_keywords: + - "192.168." + - "10." + - ".senmesh.com" + +yara_hash_whitelist: + # YARA 命中后，文件 MD5 在此列表 → 跳过告警 + # 典型场景：已知安全的开源工具被 YARA 误匹配 + - md5: "00000000000000000000000000000000" + note: "示例：占位，生产按需补充" + # ---------- 审计 ---------- audit: # 记录扫描结果明细到审计日志 diff --git a/tests/samples/L0-023_5layer_nested.zip b/tests/samples/L0-023_5layer_nested.zip index 2bfad45597d880c0f78a0f74efcef27fad9c3a1c..1726d8b498018fd684231e7a328b3adcf71f1312 100644 GIT binary patch literal 952 zcmWIWW@Zs#U|`^2m{k%R+dk*4%04Cr1_5RU240|OVo`ENW?5>ysa{oP0r!WrG{MA# zga-*TLWQKY-+j3r)6?0wkfo!taZ#u1Ny(Y1D?h4Lh0bKXBY)`+$trw54ag~dGH2ar+ z#oaBZ())Y#@86gpSeEqPsxJ4d?|=R9+OOY=toFV;v3VW8@2AhtZ$@3eyZh7noZS77 zN?QNN?mvI~dRV>osa;IqM9==<7?epL4$D7aI{a#=F;eT<(-OoR_ zp3c9gcP>8r)6OT$J2w7(eEFYc-TRz-^_S}}Uip9b+mlJmVcP2P*S_C1dH8VI?pK$l zD*m^U{k}iVudZ%M;^7`q?bFIr=FYr+`);h5=)U82?|+{9`|Q!y&Gu}w{#?EH|J~Eq zYxh5Xs&}Vm+ry`QufD{~m*?CJiv9Al|^y)opCJ%N;?Q}Nwxx_e&FIi5Hz>({1gerdm!Trz0hn_P4ueevn}(p=6>Ox<~w_WOli%J06u z;D6Ds-p#q3lk1PJUiEX28t0kKeDeY0OjhJEF)&)7?p3n(1)oDkX^K;+UxT@}v_~5! zhj(L%wr~eKLjX!bFA5d^E(J{B5Uy6IG++gq!N?@SfIGVY4FZD(Mi7OPdC;|@I{>1QfuVu%B{2UXJ14-Kl?^1z O1caqPS{Il%85jUAxT|si literal 640 zcmWIWW@Zs#00G_D*w`KJmHT;^7#Kj96DXFMmzP?kSCv^1fTHZktnVf>7#SEq7>_dl z%2(f1fJ#6ZkFrRKUq3ehEdyaZ%Cz@>=(hnX0b!7`SLd(KP6ba7 z@MdHZVL$``x+Oq+z@P!>1tcp2yjj^m0*pXt4w3+I7|;zv_ZV0akboG56t<{_T?eZ} kHw-;6i8Ra`>~M6$&?AsY!}en}3_Z~hX_zrEjWI9)0BO#AOaK4?