456 lines
14 KiB
Python
456 lines
14 KiB
Python
|
"""诊断套件持久化存储模块。
|
|||
|
|
|
|||
|
|
职责：
|
|||
|
|
- 定时器规则结果 → JSON Lines 追加（rules_history.jsonl）
|
|||
|
|
- Docker 日志快照 → 存文件（logs/*.log）
|
|||
|
|
- 自动清理过期数据（基于配置的 history_retention_days）
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import logging
|
|||
|
|
import os
|
|||
|
|
import time
|
|||
|
|
from datetime import datetime, timedelta, timezone
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Any, Dict, List, Optional
|
|||
|
|
|
|||
|
|
from app.diagnostic.config import get_diag_config
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _storage_dir() -> Path:
|
|||
|
|
"""获取存储根目录。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
p = Path(cfg.storage_dir)
|
|||
|
|
p.mkdir(parents=True, exist_ok=True)
|
|||
|
|
return p
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _logs_dir() -> Path:
|
|||
|
|
p = _storage_dir() / "logs"
|
|||
|
|
p.mkdir(parents=True, exist_ok=True)
|
|||
|
|
return p
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _rules_history_file() -> Path:
|
|||
|
|
return _storage_dir() / "rules_history.jsonl"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# 跨进程文件锁写入
|
|||
|
|
#
|
|||
|
|
# 多 worker 部署时，每个 worker 都会跑诊断定时器。"a" 追加模式在多进程
|
|||
|
|
# 下不安全——各进程独立维护文件偏移量，后写的会覆盖先写的内容，把一行
|
|||
|
|
# JSON 从中间劈开，产出非法 UTF-8 字节，导致后续读取全部报错。
|
|||
|
|
# 这里用 fcntl 文件锁保证同一时刻只有一个进程在写。
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
import fcntl # type: ignore
|
|||
|
|
|
|||
|
|
_HAS_FCNTL = True
|
|||
|
|
except ImportError: # pragma: no cover - Windows 平台
|
|||
|
|
fcntl = None # type: ignore
|
|||
|
|
_HAS_FCNTL = False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _append_line_locked(path: Path, line: str) -> None:
|
|||
|
|
"""向文件追加一行，跨进程互斥。
|
|||
|
|
|
|||
|
|
Linux 下用 fcntl.flock 排他锁；无 fcntl 时退化为进程内线程锁。
|
|||
|
|
"""
|
|||
|
|
payload = line + "\n"
|
|||
|
|
with open(path, "a", encoding="utf-8") as fp:
|
|||
|
|
if _HAS_FCNTL:
|
|||
|
|
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
|
|||
|
|
try:
|
|||
|
|
fp.seek(0, os.SEEK_END)
|
|||
|
|
fp.write(payload)
|
|||
|
|
fp.flush()
|
|||
|
|
os.fsync(fp.fileno())
|
|||
|
|
finally:
|
|||
|
|
if _HAS_FCNTL:
|
|||
|
|
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _rewrite_file_locked(path: Path, lines: List[str]) -> None:
|
|||
|
|
"""整文件重写（清理过期数据用），同样走文件锁。
|
|||
|
|
|
|||
|
|
写临时文件再原子替换，避免重写过程中其他进程读到半截内容。
|
|||
|
|
"""
|
|||
|
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
|||
|
|
with open(tmp, "w", encoding="utf-8") as fp:
|
|||
|
|
if _HAS_FCNTL:
|
|||
|
|
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
|
|||
|
|
try:
|
|||
|
|
for line in lines:
|
|||
|
|
fp.write(line + "\n")
|
|||
|
|
fp.flush()
|
|||
|
|
os.fsync(fp.fileno())
|
|||
|
|
finally:
|
|||
|
|
if _HAS_FCNTL:
|
|||
|
|
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
|
|||
|
|
os.replace(tmp, path)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# 规则历史：JSON Lines 追加
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def append_rule_results(
|
|||
|
|
rules: List[Any],
|
|||
|
|
timer_name: str,
|
|||
|
|
) -> None:
|
|||
|
|
"""定时器每次跑完追加一行 JSONL。
|
|||
|
|
|
|||
|
|
每行格式::
|
|||
|
|
|
|||
|
|
{"ts": "2026-10-28T13:12:14", "timer": "error_scan",
|
|||
|
|
"docker_status": "full",
|
|||
|
|
"rules": [{"id":"R001","severity":"ok","message":"...","run_at":"...","run_by":"timer:error_scan",...}]}
|
|||
|
|
"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 北京时间
|
|||
|
|
bj = timezone(timedelta(hours=8))
|
|||
|
|
entry: Dict[str, Any] = {
|
|||
|
|
"ts": datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S"),
|
|||
|
|
"timer": timer_name,
|
|||
|
|
"docker_status": None,
|
|||
|
|
"rules": [
|
|||
|
|
r.to_dict() if hasattr(r, "to_dict") else dict(r)
|
|||
|
|
for r in rules
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
f = _rules_history_file()
|
|||
|
|
_append_line_locked(f, json.dumps(entry, ensure_ascii=False))
|
|||
|
|
|
|||
|
|
# 顺带清理过期
|
|||
|
|
_trim_history()
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("诊断持久化写入失败: %s", e)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_rule_history(days: Optional[int] = None) -> List[Dict[str, Any]]:
|
|||
|
|
"""读取规则历史 JSONL，返回最近 N 天的行。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return []
|
|||
|
|
days = days or cfg.history_retention_days
|
|||
|
|
|
|||
|
|
f = _rules_history_file()
|
|||
|
|
if not f.exists():
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|||
|
|
results: List[Dict[str, Any]] = []
|
|||
|
|
try:
|
|||
|
|
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
|||
|
|
for line in fp:
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
obj = json.loads(line)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
continue
|
|||
|
|
# 按 ts 判断是否过期
|
|||
|
|
ts_str = obj.get("ts", "")
|
|||
|
|
try:
|
|||
|
|
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
|||
|
|
if ts_dt >= cutoff:
|
|||
|
|
results.append(obj)
|
|||
|
|
except ValueError:
|
|||
|
|
# 时间格式不对就不过滤，保留
|
|||
|
|
results.append(obj)
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("诊断历史读取失败: %s", e)
|
|||
|
|
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _trim_history() -> None:
|
|||
|
|
"""清理过期的规则历史行（保留文件头）。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
days = cfg.history_retention_days
|
|||
|
|
|
|||
|
|
f = _rules_history_file()
|
|||
|
|
if not f.exists():
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|||
|
|
kept: List[str] = []
|
|||
|
|
try:
|
|||
|
|
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
|||
|
|
total = 0
|
|||
|
|
for line in fp:
|
|||
|
|
total += 1
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
obj = json.loads(line)
|
|||
|
|
ts_str = obj.get("ts", "")
|
|||
|
|
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
|||
|
|
if ts_dt >= cutoff:
|
|||
|
|
kept.append(line)
|
|||
|
|
except Exception:
|
|||
|
|
kept.append(line) # 格式异常的保留
|
|||
|
|
|
|||
|
|
# 如果删了一些（不是全部），就重写
|
|||
|
|
if kept and len(kept) < total:
|
|||
|
|
_rewrite_file_locked(f, kept)
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("诊断历史清理失败: %s", e)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# Docker 日志快照
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def snapshot_container_logs(
|
|||
|
|
containers: List[str],
|
|||
|
|
tail: int,
|
|||
|
|
filename_prefix: Optional[str] = None,
|
|||
|
|
) -> List[Path]:
|
|||
|
|
"""把各容器当前 docker logs --tail N 存一份文件。
|
|||
|
|
|
|||
|
|
文件名: {container}-{prefix}{timestamp}.log
|
|||
|
|
prefix 可选（如 timer 存的加前缀 timer-）
|
|||
|
|
返回写入的文件列表。
|
|||
|
|
"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
from app.diagnostic.docker_probe import get_client
|
|||
|
|
|
|||
|
|
client = get_client()
|
|||
|
|
if client is None:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|||
|
|
prefix = f"{filename_prefix}-" if filename_prefix else ""
|
|||
|
|
out: List[Path] = []
|
|||
|
|
|
|||
|
|
for name in containers:
|
|||
|
|
try:
|
|||
|
|
c = client.containers.get(name)
|
|||
|
|
raw = c.logs(tail=tail)
|
|||
|
|
text = (
|
|||
|
|
raw.decode("utf-8", errors="replace")
|
|||
|
|
if isinstance(raw, (bytes, bytearray))
|
|||
|
|
else str(raw)
|
|||
|
|
)
|
|||
|
|
fname = f"{name}-{prefix}{ts}.log"
|
|||
|
|
fpath = _logs_dir() / fname
|
|||
|
|
fpath.write_text(text, encoding="utf-8")
|
|||
|
|
out.append(fpath)
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("容器 %s 日志快照失败: %s", name, e)
|
|||
|
|
|
|||
|
|
# 顺带清理过期日志
|
|||
|
|
_trim_logs()
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_log_snapshots(containers: Optional[List[str]] = None) -> Dict[str, List[Path]]:
|
|||
|
|
"""读 logs/ 目录，返回 {container_name: [path, path, ...]}。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
cutoff = datetime.now() - timedelta(days=cfg.history_retention_days)
|
|||
|
|
result: Dict[str, List[Path]] = {}
|
|||
|
|
for f in _logs_dir().glob("*.log"):
|
|||
|
|
stem = f.stem # 去掉 .log
|
|||
|
|
# 文件名格式: meshsafe-api-timer-20261028-1312 或 meshsafe-api-20261028-1312
|
|||
|
|
# 容器名是 stem 里第一个 - 前面的部分... 其实容器名本身也可能带 -
|
|||
|
|
# 简化：直接用 stem 整体当 key（导出时按需匹配）
|
|||
|
|
# 但我们需要按容器分组，所以用 glob 前缀匹配
|
|||
|
|
if containers:
|
|||
|
|
matched = None
|
|||
|
|
for c in containers:
|
|||
|
|
if stem.startswith(c + "-"):
|
|||
|
|
matched = c
|
|||
|
|
break
|
|||
|
|
if matched is None:
|
|||
|
|
continue
|
|||
|
|
container_key = matched
|
|||
|
|
else:
|
|||
|
|
# 没指定容器，尝试从文件名推断
|
|||
|
|
parts = stem.split("-")
|
|||
|
|
container_key = parts[0] if parts else stem
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
mtime = datetime.fromtimestamp(f.stat().st_mtime)
|
|||
|
|
if mtime >= cutoff:
|
|||
|
|
result.setdefault(container_key, []).append(f)
|
|||
|
|
except Exception:
|
|||
|
|
result.setdefault(container_key, []).append(f)
|
|||
|
|
|
|||
|
|
# 按 mtime 排序（新的在前）
|
|||
|
|
for k in result:
|
|||
|
|
result[k].sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
|||
|
|
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _trim_logs() -> None:
|
|||
|
|
"""清理过期的日志快照文件。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
cutoff = datetime.now() - timedelta(days=cfg.history_retention_days)
|
|||
|
|
try:
|
|||
|
|
for f in _logs_dir().glob("*.log"):
|
|||
|
|
try:
|
|||
|
|
mtime = datetime.fromtimestamp(f.stat().st_mtime)
|
|||
|
|
if mtime < cutoff:
|
|||
|
|
f.unlink()
|
|||
|
|
except FileNotFoundError:
|
|||
|
|
pass
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("日志快照清理失败: %s", e)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# 容器重启事件追踪
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def _restart_events_file() -> Path:
|
|||
|
|
return _storage_dir() / "restart_events.jsonl"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def append_restart_event(
|
|||
|
|
container: str,
|
|||
|
|
restart_before: int,
|
|||
|
|
restart_after: int,
|
|||
|
|
uptime_before: str = "",
|
|||
|
|
uptime_before_seconds: float = 0,
|
|||
|
|
trigger: str = "restart_count_increased",
|
|||
|
|
) -> None:
|
|||
|
|
"""记录一条容器重启事件到 restart_events.jsonl。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return
|
|||
|
|
try:
|
|||
|
|
bj = timezone(timedelta(hours=8))
|
|||
|
|
entry = {
|
|||
|
|
"ts": datetime.now(bj).strftime("%Y-%m-%d %H:%M:%S"),
|
|||
|
|
"container": container,
|
|||
|
|
"restart_before": restart_before,
|
|||
|
|
"restart_after": restart_after,
|
|||
|
|
"uptime_before": uptime_before,
|
|||
|
|
"uptime_before_seconds": int(uptime_before_seconds),
|
|||
|
|
"trigger": trigger,
|
|||
|
|
}
|
|||
|
|
f = _restart_events_file()
|
|||
|
|
_append_line_locked(f, json.dumps(entry, ensure_ascii=False))
|
|||
|
|
|
|||
|
|
# 清理过期事件
|
|||
|
|
_trim_restart_events()
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("重启事件写入失败: %s", e)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def snapshot_on_restart(container: str, tail: int = 300) -> Optional[Path]:
|
|||
|
|
"""容器刚重启后，抓一份当前 docker logs 片段存到 logs/*-restart-*.log。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return None
|
|||
|
|
try:
|
|||
|
|
from app.diagnostic.docker_probe import get_client
|
|||
|
|
client = get_client()
|
|||
|
|
if client is None:
|
|||
|
|
return None
|
|||
|
|
c = client.containers.get(container)
|
|||
|
|
raw = c.logs(tail=tail)
|
|||
|
|
text = (
|
|||
|
|
raw.decode("utf-8", errors="replace")
|
|||
|
|
if isinstance(raw, (bytes, bytearray))
|
|||
|
|
else str(raw)
|
|||
|
|
)
|
|||
|
|
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|||
|
|
fname = f"{container}-restart-{ts}.log"
|
|||
|
|
fpath = _logs_dir() / fname
|
|||
|
|
fpath.write_text(text, encoding="utf-8")
|
|||
|
|
return fpath
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("重启日志快照失败 %s: %s", container, e)
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_restart_events(days: Optional[int] = None) -> List[Dict[str, Any]]:
|
|||
|
|
"""读取重启事件 JSONL，返回最近 N 天的行。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.storage_enabled:
|
|||
|
|
return []
|
|||
|
|
days = days or cfg.history_retention_days
|
|||
|
|
|
|||
|
|
f = _restart_events_file()
|
|||
|
|
if not f.exists():
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|||
|
|
results: List[Dict[str, Any]] = []
|
|||
|
|
try:
|
|||
|
|
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
|||
|
|
for line in fp:
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
obj = json.loads(line)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
continue
|
|||
|
|
ts_str = obj.get("ts", "")
|
|||
|
|
try:
|
|||
|
|
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
|||
|
|
if ts_dt >= cutoff:
|
|||
|
|
results.append(obj)
|
|||
|
|
except ValueError:
|
|||
|
|
results.append(obj)
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("重启事件读取失败: %s", e)
|
|||
|
|
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _trim_restart_events() -> None:
|
|||
|
|
"""清理过期的重启事件行。"""
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
days = cfg.history_retention_days
|
|||
|
|
f = _restart_events_file()
|
|||
|
|
if not f.exists():
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|||
|
|
kept: List[str] = []
|
|||
|
|
try:
|
|||
|
|
with open(f, "r", encoding="utf-8", errors="replace") as fp:
|
|||
|
|
total = 0
|
|||
|
|
for line in fp:
|
|||
|
|
total += 1
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
obj = json.loads(line)
|
|||
|
|
ts_str = obj.get("ts", "")
|
|||
|
|
ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S")
|
|||
|
|
if ts_dt >= cutoff:
|
|||
|
|
kept.append(line)
|
|||
|
|
except Exception:
|
|||
|
|
kept.append(line)
|
|||
|
|
|
|||
|
|
if kept and len(kept) < total:
|
|||
|
|
_rewrite_file_locked(f, kept)
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
logger.warning("重启事件清理失败: %s", e)
|