339 lines
10 KiB
Python
339 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
r"""
|
||
CM95 Production 失败汇总审计 v1
|
||
|
||
用途
|
||
====
|
||
对 production_run_v1.sqlite 中 FAILED 任务做一次性归类，不修改任何状态。
|
||
|
||
默认 release：
|
||
build/releases/cm95-d1-bfa67dd5625f-98f21909a3d1
|
||
|
||
输出：
|
||
- cm95_production_failure_audit_v1.json
|
||
- cm95_production_failure_tiles_v1.csv
|
||
- cm95_production_failure_audit_v1.txt
|
||
|
||
重点统计：
|
||
1. FAILED 数量 / attempts 分布；
|
||
2. hard-gate unmapped 按 object_class 汇总；
|
||
3. MVT encoder dropped 按 layer 汇总；
|
||
4. 其它 RuntimeError / Exception 首行归类；
|
||
5. 从对应 report JSON 补充 unmapped_class_counts、
|
||
encoder_dropped_features、quantization_fallbacks；
|
||
6. 按 zoom 统计失败分布。
|
||
|
||
只读：
|
||
- 不改 SQLite；
|
||
- 不改 report；
|
||
- 不删除 PBF；
|
||
- 不重置 attempts。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import ast
|
||
import csv
|
||
import json
|
||
import re
|
||
import sqlite3
|
||
from collections import Counter, defaultdict
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
HG_RE = re.compile(r"hard-gate unmapped:\s*(\[[^\n]*\])")
|
||
ENC_RE = re.compile(
|
||
r"MVT encoder dropped features in tile\s+(\d+)/(\d+)/(\d+):\s*(\[[^\n]*\])"
|
||
)
|
||
|
||
|
||
def parse_args():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument(
|
||
"--release-dir",
|
||
default=r"build\releases\cm95-d1-bfa67dd5625f-98f21909a3d1",
|
||
)
|
||
p.add_argument("--profile", default="delivery")
|
||
p.add_argument("--output-dir", default=None)
|
||
return p.parse_args()
|
||
|
||
|
||
def safe_literal(text: str) -> Any:
|
||
try:
|
||
return ast.literal_eval(text)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def read_json(path: Path) -> dict[str, Any]:
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
obj = json.loads(path.read_text(encoding="utf-8"))
|
||
return obj if isinstance(obj, dict) else {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def classify_error(last_error: str):
|
||
first = (last_error or "").splitlines()[0].strip()
|
||
|
||
hg = HG_RE.search(first)
|
||
if hg:
|
||
items = safe_literal(hg.group(1))
|
||
return "HARD_GATE_UNMAPPED", items if isinstance(items, list) else []
|
||
|
||
enc = ENC_RE.search(first)
|
||
if enc:
|
||
items = safe_literal(enc.group(4))
|
||
return "ENCODER_DROPPED", items if isinstance(items, list) else []
|
||
|
||
if first.startswith("RuntimeError:"):
|
||
return "RUNTIME_OTHER", []
|
||
if first:
|
||
return "OTHER_EXCEPTION", []
|
||
return "NO_ERROR_TEXT", []
|
||
|
||
|
||
def main():
|
||
a = parse_args()
|
||
release_dir = Path(a.release_dir).resolve()
|
||
ledger_path = release_dir / "production_run_v1.sqlite"
|
||
if not ledger_path.exists():
|
||
raise SystemExit(f"找不到账本: {ledger_path}")
|
||
|
||
outdir = (
|
||
Path(a.output_dir).resolve()
|
||
if a.output_dir
|
||
else release_dir
|
||
)
|
||
outdir.mkdir(parents=True, exist_ok=True)
|
||
|
||
conn = sqlite3.connect(f"file:{ledger_path.as_posix()}?mode=ro", uri=True)
|
||
conn.row_factory = sqlite3.Row
|
||
try:
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT profile,z,x,y,state,attempts,updated_at,
|
||
output_feature_count,output_layer_count,written,last_error
|
||
FROM tile_job
|
||
WHERE profile=? AND state='FAILED'
|
||
ORDER BY z,x,y
|
||
""",
|
||
(a.profile,),
|
||
).fetchall()
|
||
finally:
|
||
conn.close()
|
||
|
||
kind_counts = Counter()
|
||
zoom_counts = Counter()
|
||
attempts_counts = Counter()
|
||
hard_gate_classes = Counter()
|
||
encoder_layers = Counter()
|
||
first_line_counts = Counter()
|
||
|
||
report_hard_gate_classes = Counter()
|
||
report_unmapped_classes = Counter()
|
||
report_encoder_layers = Counter()
|
||
report_quantization_layers = Counter()
|
||
|
||
detail_rows = []
|
||
missing_reports = 0
|
||
|
||
for r in rows:
|
||
z, x, y = int(r["z"]), int(r["x"]), int(r["y"])
|
||
error = str(r["last_error"] or "")
|
||
first = error.splitlines()[0].strip()
|
||
kind, parsed = classify_error(error)
|
||
|
||
kind_counts[kind] += 1
|
||
zoom_counts[z] += 1
|
||
attempts_counts[int(r["attempts"])] += 1
|
||
first_line_counts[first] += 1
|
||
|
||
if kind == "HARD_GATE_UNMAPPED":
|
||
for item in parsed:
|
||
if isinstance(item, dict):
|
||
cls = str(item.get("object_class") or "")
|
||
cnt = int(item.get("count") or 0)
|
||
if cls:
|
||
hard_gate_classes[cls] += cnt
|
||
|
||
if kind == "ENCODER_DROPPED":
|
||
for item in parsed:
|
||
if isinstance(item, dict):
|
||
layer = str(item.get("layer") or "")
|
||
if layer:
|
||
encoder_layers[layer] += 1
|
||
|
||
report_path = (
|
||
release_dir
|
||
/ "reports"
|
||
/ a.profile
|
||
/ str(z)
|
||
/ str(x)
|
||
/ f"{y}.json"
|
||
)
|
||
report = read_json(report_path)
|
||
if not report:
|
||
missing_reports += 1
|
||
|
||
for item in report.get("hard_gate_unmapped", []) or []:
|
||
if isinstance(item, dict):
|
||
cls = str(item.get("object_class") or "")
|
||
cnt = int(item.get("count") or 0)
|
||
if cls:
|
||
report_hard_gate_classes[cls] += cnt
|
||
|
||
for cls, cnt in (report.get("unmapped_class_counts") or {}).items():
|
||
report_unmapped_classes[str(cls)] += int(cnt or 0)
|
||
|
||
for item in report.get("encoder_dropped_features", []) or []:
|
||
if isinstance(item, dict):
|
||
layer = str(item.get("layer") or "")
|
||
if layer:
|
||
report_encoder_layers[layer] += 1
|
||
|
||
for item in report.get("quantization_fallbacks", []) or []:
|
||
if isinstance(item, dict):
|
||
layer = str(
|
||
item.get("from_layer")
|
||
or item.get("layer")
|
||
or "UNKNOWN"
|
||
)
|
||
report_quantization_layers[layer] += 1
|
||
|
||
detail_rows.append(
|
||
{
|
||
"profile": a.profile,
|
||
"z": z,
|
||
"x": x,
|
||
"y": y,
|
||
"attempts": int(r["attempts"]),
|
||
"failure_kind": kind,
|
||
"first_error_line": first,
|
||
"report_exists": report_path.exists(),
|
||
"hard_gate_unmapped": report.get("hard_gate_unmapped", []),
|
||
"unmapped_class_counts": report.get("unmapped_class_counts", {}),
|
||
"encoder_dropped_features": report.get("encoder_dropped_features", []),
|
||
"quantization_fallbacks": report.get("quantization_fallbacks", []),
|
||
"report_path": str(report_path),
|
||
}
|
||
)
|
||
|
||
report = {
|
||
"schema": "navsea.cm95.production-failure-audit/v1",
|
||
"release_dir": str(release_dir),
|
||
"profile": a.profile,
|
||
"failed_jobs": len(rows),
|
||
"missing_reports": missing_reports,
|
||
"failure_kind_counts": dict(sorted(kind_counts.items())),
|
||
"zoom_counts": {str(k): v for k, v in sorted(zoom_counts.items())},
|
||
"attempts_counts": {str(k): v for k, v in sorted(attempts_counts.items())},
|
||
"hard_gate_classes_from_error": dict(
|
||
sorted(hard_gate_classes.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
),
|
||
"hard_gate_classes_from_report": dict(
|
||
sorted(report_hard_gate_classes.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
),
|
||
"all_unmapped_classes_from_report": dict(
|
||
sorted(report_unmapped_classes.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
),
|
||
"encoder_dropped_layers_from_error": dict(
|
||
sorted(encoder_layers.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
),
|
||
"encoder_dropped_layers_from_report": dict(
|
||
sorted(report_encoder_layers.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
),
|
||
"quantization_fallback_layers_from_report": dict(
|
||
sorted(report_quantization_layers.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
),
|
||
"distinct_first_error_lines": [
|
||
{"error": k, "tiles": v}
|
||
for k, v in first_line_counts.most_common()
|
||
],
|
||
"tiles": detail_rows,
|
||
}
|
||
|
||
json_path = outdir / "cm95_production_failure_audit_v1.json"
|
||
csv_path = outdir / "cm95_production_failure_tiles_v1.csv"
|
||
txt_path = outdir / "cm95_production_failure_audit_v1.txt"
|
||
|
||
json_path.write_text(
|
||
json.dumps(report, ensure_ascii=False, indent=2, default=str),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
with csv_path.open("w", encoding="utf-8-sig", newline="") as f:
|
||
fields = [
|
||
"profile", "z", "x", "y", "attempts",
|
||
"failure_kind", "first_error_line",
|
||
"report_exists", "hard_gate_unmapped",
|
||
"unmapped_class_counts",
|
||
"encoder_dropped_features",
|
||
"quantization_fallbacks",
|
||
"report_path",
|
||
]
|
||
w = csv.DictWriter(f, fieldnames=fields)
|
||
w.writeheader()
|
||
for row in detail_rows:
|
||
out = dict(row)
|
||
for key in (
|
||
"hard_gate_unmapped",
|
||
"unmapped_class_counts",
|
||
"encoder_dropped_features",
|
||
"quantization_fallbacks",
|
||
):
|
||
out[key] = json.dumps(
|
||
out[key],
|
||
ensure_ascii=False,
|
||
separators=(",", ":"),
|
||
)
|
||
w.writerow(out)
|
||
|
||
lines = [
|
||
"CM95 Production 失败汇总审计 v1",
|
||
"=" * 72,
|
||
f"Release : {release_dir}",
|
||
f"Profile : {a.profile}",
|
||
f"FAILED : {len(rows)}",
|
||
f"Missing reports : {missing_reports}",
|
||
"",
|
||
"Failure kinds:",
|
||
]
|
||
for k, v in kind_counts.most_common():
|
||
lines.append(f" {k:<24} {v}")
|
||
|
||
lines += ["", "Hard-gate classes:"]
|
||
if report_hard_gate_classes:
|
||
for k, v in report_hard_gate_classes.most_common():
|
||
lines.append(f" {k:<16} {v}")
|
||
else:
|
||
lines.append(" (none)")
|
||
|
||
lines += ["", "Encoder dropped layers:"]
|
||
if report_encoder_layers:
|
||
for k, v in report_encoder_layers.most_common():
|
||
lines.append(f" {k:<28} {v}")
|
||
else:
|
||
lines.append(" (none)")
|
||
|
||
lines += ["", "FAILED by zoom:"]
|
||
for z, v in sorted(zoom_counts.items()):
|
||
lines.append(f" z{z:<2} {v}")
|
||
|
||
txt_path.write_text("\n".join(lines), encoding="utf-8")
|
||
|
||
print("\n".join(lines))
|
||
print("")
|
||
print(json_path)
|
||
print(csv_path)
|
||
print(txt_path)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|