286 lines
10 KiB
Python
286 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
p = argparse.ArgumentParser(
|
|
description="Audit CM95 navigation composite name conflicts from production trace reports."
|
|
)
|
|
p.add_argument("--release-dir", required=True, type=Path)
|
|
p.add_argument(
|
|
"--profile",
|
|
default="delivery",
|
|
choices=["delivery", "engineering"],
|
|
)
|
|
p.add_argument("--output-json", required=True, type=Path)
|
|
p.add_argument("--output-csv", required=True, type=Path)
|
|
return p.parse_args()
|
|
|
|
|
|
def tile_from_trace_path(trace_path: Path, trace_root: Path) -> str:
|
|
rel = trace_path.relative_to(trace_root)
|
|
parts = rel.parts
|
|
if len(parts) < 3:
|
|
return rel.as_posix()
|
|
z, x, y_name = parts[-3], parts[-2], parts[-1]
|
|
y = y_name.replace(".trace.jsonl", "")
|
|
return f"{z}/{x}/{y}"
|
|
|
|
|
|
def read_jsonl(path: Path):
|
|
with path.open("r", encoding="utf-8") as f:
|
|
for line_no, raw in enumerate(f, 1):
|
|
raw = raw.strip()
|
|
if not raw:
|
|
continue
|
|
try:
|
|
obj = json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
|
|
if isinstance(obj, dict):
|
|
yield obj
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
release_dir = args.release_dir.resolve()
|
|
report_root = release_dir / "reports" / args.profile
|
|
|
|
if not report_root.exists():
|
|
raise SystemExit(f"report root not found: {report_root}")
|
|
|
|
trace_files = sorted(report_root.rglob("*.trace.jsonl"))
|
|
report_files = sorted(
|
|
p for p in report_root.rglob("*.json")
|
|
if not p.name.endswith(".trace.json")
|
|
)
|
|
|
|
report_conflict_occurrences = 0
|
|
for path in report_files:
|
|
try:
|
|
obj = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
continue
|
|
report_conflict_occurrences += int(
|
|
(obj.get("navigation_composites") or {}).get("name_conflict_groups") or 0
|
|
)
|
|
|
|
raw_conflicts: list[dict[str, Any]] = []
|
|
|
|
for trace_path in trace_files:
|
|
tile = tile_from_trace_path(trace_path, report_root)
|
|
z = int(tile.split("/", 1)[0]) if "/" in tile else None
|
|
|
|
for row in read_jsonl(trace_path):
|
|
if row.get("nav_name_conflict") is not True:
|
|
continue
|
|
|
|
candidates = [
|
|
str(v) for v in (row.get("nav_name_candidates") or [])
|
|
if v is not None and str(v) != ""
|
|
]
|
|
components = [
|
|
str(v) for v in (row.get("source_component_ids") or [])
|
|
if v is not None
|
|
]
|
|
|
|
raw_conflicts.append(
|
|
{
|
|
"tile": tile,
|
|
"zoom": z,
|
|
"output_id": row.get("output_id"),
|
|
"output_layer": row.get("output_layer"),
|
|
"source_id": row.get("source_id"),
|
|
"relation20": row.get("relation20"),
|
|
"native_scale": row.get("native_scale"),
|
|
"mapping_rule_id": row.get("mapping_rule_id"),
|
|
"mapping_status": row.get("mapping_status"),
|
|
"nav_name_source": row.get("nav_name_source"),
|
|
"nav_name_candidates": candidates,
|
|
"source_component_ids": components,
|
|
"trace_path": str(trace_path.relative_to(release_dir)).replace("\\", "/"),
|
|
}
|
|
)
|
|
|
|
unique: dict[str, dict[str, Any]] = {}
|
|
|
|
for item in raw_conflicts:
|
|
output_id = item.get("output_id")
|
|
if output_id is not None:
|
|
key = f"output_id:{output_id}"
|
|
else:
|
|
key = json.dumps(
|
|
{
|
|
"source_id": item.get("source_id"),
|
|
"relation20": item.get("relation20"),
|
|
"native_scale": item.get("native_scale"),
|
|
"components": sorted(item.get("source_component_ids") or []),
|
|
},
|
|
sort_keys=True,
|
|
ensure_ascii=False,
|
|
)
|
|
|
|
if key not in unique:
|
|
unique[key] = {
|
|
"key": key,
|
|
"output_id": item.get("output_id"),
|
|
"output_layer": item.get("output_layer"),
|
|
"source_id": item.get("source_id"),
|
|
"relation20": item.get("relation20"),
|
|
"native_scale": item.get("native_scale"),
|
|
"mapping_rule_id": item.get("mapping_rule_id"),
|
|
"mapping_status": item.get("mapping_status"),
|
|
"nav_name_candidates": list(item.get("nav_name_candidates") or []),
|
|
"source_component_ids": list(item.get("source_component_ids") or []),
|
|
"occurrences": 0,
|
|
"tiles": [],
|
|
"zooms": [],
|
|
"trace_paths": [],
|
|
}
|
|
|
|
u = unique[key]
|
|
u["occurrences"] += 1
|
|
|
|
tile = item["tile"]
|
|
if tile not in u["tiles"]:
|
|
u["tiles"].append(tile)
|
|
|
|
z = item.get("zoom")
|
|
if z is not None and z not in u["zooms"]:
|
|
u["zooms"].append(z)
|
|
|
|
tpath = item["trace_path"]
|
|
if tpath not in u["trace_paths"]:
|
|
u["trace_paths"].append(tpath)
|
|
|
|
# Preserve any candidates/components that differ across repeated tile occurrences.
|
|
for candidate in item.get("nav_name_candidates") or []:
|
|
if candidate not in u["nav_name_candidates"]:
|
|
u["nav_name_candidates"].append(candidate)
|
|
for component in item.get("source_component_ids") or []:
|
|
if component not in u["source_component_ids"]:
|
|
u["source_component_ids"].append(component)
|
|
|
|
unique_rows = list(unique.values())
|
|
|
|
for row in unique_rows:
|
|
row["tiles"].sort()
|
|
row["zooms"].sort()
|
|
row["trace_paths"].sort()
|
|
|
|
unique_rows.sort(
|
|
key=lambda r: (
|
|
-(int(r.get("occurrences") or 0)),
|
|
int(r.get("native_scale") or 0),
|
|
str(r.get("source_id") or ""),
|
|
)
|
|
)
|
|
|
|
zoom_occurrences = Counter(
|
|
int(x["zoom"]) for x in raw_conflicts if x.get("zoom") is not None
|
|
)
|
|
scale_unique = Counter(
|
|
int(x["native_scale"]) for x in unique_rows if x.get("native_scale") is not None
|
|
)
|
|
candidate_count_unique = Counter(
|
|
len(x.get("nav_name_candidates") or []) for x in unique_rows
|
|
)
|
|
|
|
tile_set = {x["tile"] for x in raw_conflicts}
|
|
|
|
result = {
|
|
"schema": "navsea.cm95.nav-name-conflict-audit/v1",
|
|
"release_dir": str(release_dir),
|
|
"profile": args.profile,
|
|
"trace_file_count": len(trace_files),
|
|
"report_file_count": len(report_files),
|
|
"report_name_conflict_occurrences": report_conflict_occurrences,
|
|
"trace_name_conflict_occurrences": len(raw_conflicts),
|
|
"unique_conflict_outputs": len(unique_rows),
|
|
"affected_tiles": len(tile_set),
|
|
"occurrences_by_zoom": dict(sorted(zoom_occurrences.items())),
|
|
"unique_conflicts_by_native_scale": dict(sorted(scale_unique.items())),
|
|
"unique_conflicts_by_candidate_count": dict(sorted(candidate_count_unique.items())),
|
|
"unique_conflicts": unique_rows,
|
|
}
|
|
|
|
args.output_json.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output_json.write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
args.output_csv.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.output_csv.open("w", encoding="utf-8-sig", newline="") as f:
|
|
writer = csv.DictWriter(
|
|
f,
|
|
fieldnames=[
|
|
"output_id",
|
|
"source_id",
|
|
"relation20",
|
|
"native_scale",
|
|
"occurrences",
|
|
"tile_count",
|
|
"zooms",
|
|
"nav_name_candidates",
|
|
"source_component_ids",
|
|
"mapping_rule_id",
|
|
"mapping_status",
|
|
],
|
|
)
|
|
writer.writeheader()
|
|
for row in unique_rows:
|
|
writer.writerow(
|
|
{
|
|
"output_id": row.get("output_id"),
|
|
"source_id": row.get("source_id"),
|
|
"relation20": row.get("relation20"),
|
|
"native_scale": row.get("native_scale"),
|
|
"occurrences": row.get("occurrences"),
|
|
"tile_count": len(row.get("tiles") or []),
|
|
"zooms": "|".join(str(v) for v in row.get("zooms") or []),
|
|
"nav_name_candidates": " | ".join(row.get("nav_name_candidates") or []),
|
|
"source_component_ids": " | ".join(row.get("source_component_ids") or []),
|
|
"mapping_rule_id": row.get("mapping_rule_id"),
|
|
"mapping_status": row.get("mapping_status"),
|
|
}
|
|
)
|
|
|
|
print("CM95 navigation name conflict audit")
|
|
print("=" * 72)
|
|
print(f"Release : {release_dir}")
|
|
print(f"Report conflict occurrences : {report_conflict_occurrences}")
|
|
print(f"Trace conflict occurrences : {len(raw_conflicts)}")
|
|
print(f"Unique conflict outputs : {len(unique_rows)}")
|
|
print(f"Affected tiles : {len(tile_set)}")
|
|
print(f"JSON : {args.output_json}")
|
|
print(f"CSV : {args.output_csv}")
|
|
|
|
if report_conflict_occurrences != len(raw_conflicts):
|
|
print("")
|
|
print("WARNING: report count and trace count differ; inspect reports before changing policy.")
|
|
|
|
print("")
|
|
print("Top unique conflicts:")
|
|
for row in unique_rows[:20]:
|
|
print(
|
|
f" output_id={row.get('output_id')} "
|
|
f"scale={row.get('native_scale')} "
|
|
f"occ={row.get('occurrences')} "
|
|
f"tiles={len(row.get('tiles') or [])} "
|
|
f"relation20={row.get('relation20')} "
|
|
f"names={row.get('nav_name_candidates')}"
|
|
)
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|