from __future__ import annotations import argparse import csv import json from collections import Counter, defaultdict from pathlib import Path from typing import Any def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( description="Audit CM95 navigation composite name conflicts from production trace reports." ) p.add_argument("--release-dir", required=True, type=Path) p.add_argument( "--profile", default="delivery", choices=["delivery", "engineering"], ) p.add_argument("--output-json", required=True, type=Path) p.add_argument("--output-csv", required=True, type=Path) return p.parse_args() def tile_from_trace_path(trace_path: Path, trace_root: Path) -> str: rel = trace_path.relative_to(trace_root) parts = rel.parts if len(parts) < 3: return rel.as_posix() z, x, y_name = parts[-3], parts[-2], parts[-1] y = y_name.replace(".trace.jsonl", "") return f"{z}/{x}/{y}" def read_jsonl(path: Path): with path.open("r", encoding="utf-8") as f: for line_no, raw in enumerate(f, 1): raw = raw.strip() if not raw: continue try: obj = json.loads(raw) except json.JSONDecodeError as exc: raise RuntimeError(f"{path}:{line_no}: invalid JSON: {exc}") from exc if isinstance(obj, dict): yield obj def main() -> int: args = parse_args() release_dir = args.release_dir.resolve() report_root = release_dir / "reports" / args.profile if not report_root.exists(): raise SystemExit(f"report root not found: {report_root}") trace_files = sorted(report_root.rglob("*.trace.jsonl")) report_files = sorted( p for p in report_root.rglob("*.json") if not p.name.endswith(".trace.json") ) report_conflict_occurrences = 0 for path in report_files: try: obj = json.loads(path.read_text(encoding="utf-8")) except Exception: continue report_conflict_occurrences += int( (obj.get("navigation_composites") or {}).get("name_conflict_groups") or 0 ) raw_conflicts: list[dict[str, Any]] = [] for trace_path in trace_files: tile = tile_from_trace_path(trace_path, report_root) z = int(tile.split("/", 1)[0]) if "/" in tile else None for row in read_jsonl(trace_path): if row.get("nav_name_conflict") is not True: continue candidates = [ str(v) for v in (row.get("nav_name_candidates") or []) if v is not None and str(v) != "" ] components = [ str(v) for v in (row.get("source_component_ids") or []) if v is not None ] raw_conflicts.append( { "tile": tile, "zoom": z, "output_id": row.get("output_id"), "output_layer": row.get("output_layer"), "source_id": row.get("source_id"), "relation20": row.get("relation20"), "native_scale": row.get("native_scale"), "mapping_rule_id": row.get("mapping_rule_id"), "mapping_status": row.get("mapping_status"), "nav_name_source": row.get("nav_name_source"), "nav_name_candidates": candidates, "source_component_ids": components, "trace_path": str(trace_path.relative_to(release_dir)).replace("\\", "/"), } ) unique: dict[str, dict[str, Any]] = {} for item in raw_conflicts: output_id = item.get("output_id") if output_id is not None: key = f"output_id:{output_id}" else: key = json.dumps( { "source_id": item.get("source_id"), "relation20": item.get("relation20"), "native_scale": item.get("native_scale"), "components": sorted(item.get("source_component_ids") or []), }, sort_keys=True, ensure_ascii=False, ) if key not in unique: unique[key] = { "key": key, "output_id": item.get("output_id"), "output_layer": item.get("output_layer"), "source_id": item.get("source_id"), "relation20": item.get("relation20"), "native_scale": item.get("native_scale"), "mapping_rule_id": item.get("mapping_rule_id"), "mapping_status": item.get("mapping_status"), "nav_name_candidates": list(item.get("nav_name_candidates") or []), "source_component_ids": list(item.get("source_component_ids") or []), "occurrences": 0, "tiles": [], "zooms": [], "trace_paths": [], } u = unique[key] u["occurrences"] += 1 tile = item["tile"] if tile not in u["tiles"]: u["tiles"].append(tile) z = item.get("zoom") if z is not None and z not in u["zooms"]: u["zooms"].append(z) tpath = item["trace_path"] if tpath not in u["trace_paths"]: u["trace_paths"].append(tpath) # Preserve any candidates/components that differ across repeated tile occurrences. for candidate in item.get("nav_name_candidates") or []: if candidate not in u["nav_name_candidates"]: u["nav_name_candidates"].append(candidate) for component in item.get("source_component_ids") or []: if component not in u["source_component_ids"]: u["source_component_ids"].append(component) unique_rows = list(unique.values()) for row in unique_rows: row["tiles"].sort() row["zooms"].sort() row["trace_paths"].sort() unique_rows.sort( key=lambda r: ( -(int(r.get("occurrences") or 0)), int(r.get("native_scale") or 0), str(r.get("source_id") or ""), ) ) zoom_occurrences = Counter( int(x["zoom"]) for x in raw_conflicts if x.get("zoom") is not None ) scale_unique = Counter( int(x["native_scale"]) for x in unique_rows if x.get("native_scale") is not None ) candidate_count_unique = Counter( len(x.get("nav_name_candidates") or []) for x in unique_rows ) tile_set = {x["tile"] for x in raw_conflicts} result = { "schema": "navsea.cm95.nav-name-conflict-audit/v1", "release_dir": str(release_dir), "profile": args.profile, "trace_file_count": len(trace_files), "report_file_count": len(report_files), "report_name_conflict_occurrences": report_conflict_occurrences, "trace_name_conflict_occurrences": len(raw_conflicts), "unique_conflict_outputs": len(unique_rows), "affected_tiles": len(tile_set), "occurrences_by_zoom": dict(sorted(zoom_occurrences.items())), "unique_conflicts_by_native_scale": dict(sorted(scale_unique.items())), "unique_conflicts_by_candidate_count": dict(sorted(candidate_count_unique.items())), "unique_conflicts": unique_rows, } args.output_json.parent.mkdir(parents=True, exist_ok=True) args.output_json.write_text( json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8", ) args.output_csv.parent.mkdir(parents=True, exist_ok=True) with args.output_csv.open("w", encoding="utf-8-sig", newline="") as f: writer = csv.DictWriter( f, fieldnames=[ "output_id", "source_id", "relation20", "native_scale", "occurrences", "tile_count", "zooms", "nav_name_candidates", "source_component_ids", "mapping_rule_id", "mapping_status", ], ) writer.writeheader() for row in unique_rows: writer.writerow( { "output_id": row.get("output_id"), "source_id": row.get("source_id"), "relation20": row.get("relation20"), "native_scale": row.get("native_scale"), "occurrences": row.get("occurrences"), "tile_count": len(row.get("tiles") or []), "zooms": "|".join(str(v) for v in row.get("zooms") or []), "nav_name_candidates": " | ".join(row.get("nav_name_candidates") or []), "source_component_ids": " | ".join(row.get("source_component_ids") or []), "mapping_rule_id": row.get("mapping_rule_id"), "mapping_status": row.get("mapping_status"), } ) print("CM95 navigation name conflict audit") print("=" * 72) print(f"Release : {release_dir}") print(f"Report conflict occurrences : {report_conflict_occurrences}") print(f"Trace conflict occurrences : {len(raw_conflicts)}") print(f"Unique conflict outputs : {len(unique_rows)}") print(f"Affected tiles : {len(tile_set)}") print(f"JSON : {args.output_json}") print(f"CSV : {args.output_csv}") if report_conflict_occurrences != len(raw_conflicts): print("") print("WARNING: report count and trace count differ; inspect reports before changing policy.") print("") print("Top unique conflicts:") for row in unique_rows[:20]: print( f" output_id={row.get('output_id')} " f"scale={row.get('native_scale')} " f"occ={row.get('occurrences')} " f"tiles={len(row.get('tiles') or [])} " f"relation20={row.get('relation20')} " f"names={row.get('nav_name_candidates')}" ) return 0 if __name__ == "__main__": raise SystemExit(main())