Checkpoint CM95 PBF production code and docs for Linux migration
This commit is contained in:
kaotei
328
tools/cm95_style_contract_unmatched_extract_v1.py
Normal file
328
tools/cm95_style_contract_unmatched_extract_v1.py
Normal file
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
r"""
|
||||
CM95 Style Contract unmatched Feature 精确提取器 v1
|
||||
|
||||
用途
|
||||
====
|
||||
对一个 production release 的 delivery PBF 做只读扫描，使用与
|
||||
cm95pbf.style_contract 相同的 Style filter evaluator，提取所有
|
||||
contract_unmatched Feature 的精确属性。
|
||||
|
||||
默认只关注当前已知失败的两个 source-layer：
|
||||
- place_label_land
|
||||
- seabed_text_point
|
||||
|
||||
不会修改 PBF、report、ledger 或 release manifest。
|
||||
|
||||
输出：
|
||||
build/production/cm95_style_contract_unmatched_features_v1.json
|
||||
build/production/cm95_style_contract_unmatched_features_v1.txt
|
||||
|
||||
示例：
|
||||
& ".\.venv\Scripts\python.exe" .\tools\cm95_style_contract_unmatched_extract_v1.py `
|
||||
--release-dir .\build\releases\cm95-d1-bfa67dd5625f-f5e95b1e5341 `
|
||||
--style .\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import mapbox_vector_tile
|
||||
|
||||
try:
|
||||
from cm95pbf.style_contract import ev
|
||||
except Exception:
|
||||
def ev(expr: Any, props: dict[str, Any]):
|
||||
if not isinstance(expr, list):
|
||||
return expr
|
||||
if not expr:
|
||||
return None
|
||||
op = expr[0]
|
||||
if op == "get":
|
||||
return props.get(expr[1])
|
||||
if op == "has":
|
||||
return expr[1] in props
|
||||
if op == "!":
|
||||
return not bool(ev(expr[1], props))
|
||||
if op == "all":
|
||||
return all(bool(ev(x, props)) for x in expr[1:])
|
||||
if op == "any":
|
||||
return any(bool(ev(x, props)) for x in expr[1:])
|
||||
if op == "==":
|
||||
return ev(expr[1], props) == ev(expr[2], props)
|
||||
if op == "!=":
|
||||
return ev(expr[1], props) != ev(expr[2], props)
|
||||
if op == "slice":
|
||||
value = ev(expr[1], props)
|
||||
if value is None:
|
||||
return None
|
||||
start = int(ev(expr[2], props))
|
||||
end = int(ev(expr[3], props)) if len(expr) > 3 else None
|
||||
return str(value)[start:end]
|
||||
if op == "to-number":
|
||||
value = ev(expr[1], props)
|
||||
try:
|
||||
num = float(value)
|
||||
return int(num) if num.is_integer() else num
|
||||
except Exception:
|
||||
return 0
|
||||
if op == "literal":
|
||||
return expr[1]
|
||||
if op == "match":
|
||||
value = ev(expr[1], props)
|
||||
pairs = expr[2:-1]
|
||||
default = expr[-1]
|
||||
for i in range(0, len(pairs), 2):
|
||||
label = pairs[i]
|
||||
output = pairs[i + 1]
|
||||
if isinstance(label, list) and label and label[0] == "literal":
|
||||
labels = label[1]
|
||||
if value in labels:
|
||||
return ev(output, props)
|
||||
elif isinstance(label, list):
|
||||
if value in label:
|
||||
return ev(output, props)
|
||||
elif value == label:
|
||||
return ev(output, props)
|
||||
return ev(default, props)
|
||||
raise RuntimeError(f"unsupported style filter op: {op}")
|
||||
|
||||
|
||||
DEFAULT_TARGETS = ("place_label_land", "seabed_text_point")
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--release-dir", required=True)
|
||||
p.add_argument("--style", required=True)
|
||||
p.add_argument(
|
||||
"--profile",
|
||||
default="delivery",
|
||||
help="PBF profile directory under release/pbf (default delivery)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--source-layer",
|
||||
action="append",
|
||||
dest="source_layers",
|
||||
help="可重复；默认只查 place_label_land 和 seabed_text_point",
|
||||
)
|
||||
p.add_argument(
|
||||
"--output",
|
||||
default=r"build\production\cm95_style_contract_unmatched_features_v1.json",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def compact_filter_result(style_layer: dict[str, Any], props: dict[str, Any]) -> dict[str, Any]:
|
||||
flt = style_layer.get("filter")
|
||||
matched = True if flt is None else bool(ev(flt, props))
|
||||
return {
|
||||
"style_layer_id": style_layer.get("id"),
|
||||
"filter": flt,
|
||||
"filter_matched": matched,
|
||||
"minzoom": style_layer.get("minzoom"),
|
||||
"maxzoom": style_layer.get("maxzoom"),
|
||||
}
|
||||
|
||||
|
||||
def classify_missing(source_layer: str, props: dict[str, Any]) -> dict[str, Any]:
|
||||
if source_layer == "place_label_land":
|
||||
return {
|
||||
"canonical_object_type": props.get("canonical_object_type"),
|
||||
"has_place_name_en": "place_name_en" in props,
|
||||
"place_name_en": props.get("place_name_en"),
|
||||
"has_chart_label_text": "chart_label_text" in props,
|
||||
"chart_label_text": props.get("chart_label_text"),
|
||||
"likely_issue": (
|
||||
"NO_RENDERABLE_PLACE_LABEL"
|
||||
if "place_name_en" not in props and "chart_label_text" not in props
|
||||
else "OTHER_FILTER_MISMATCH"
|
||||
),
|
||||
}
|
||||
if source_layer == "seabed_text_point":
|
||||
return {
|
||||
"canonical_object_type": props.get("canonical_object_type"),
|
||||
"has_chart_label_text": "chart_label_text" in props,
|
||||
"chart_label_text": props.get("chart_label_text"),
|
||||
"depth_value_m": props.get("depth_value_m"),
|
||||
"likely_issue": (
|
||||
"NO_RENDERABLE_SEABED_LABEL"
|
||||
if "chart_label_text" not in props
|
||||
else "OTHER_FILTER_MISMATCH"
|
||||
),
|
||||
}
|
||||
return {
|
||||
"canonical_object_type": props.get("canonical_object_type"),
|
||||
"likely_issue": "GENERIC_UNMATCHED",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
release_dir = Path(args.release_dir).resolve()
|
||||
style_path = Path(args.style).resolve()
|
||||
output_path = Path(args.output).resolve()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pbf_root = release_dir / "pbf" / args.profile
|
||||
if not pbf_root.exists():
|
||||
raise SystemExit(f"找不到 PBF 根目录: {pbf_root}")
|
||||
if not style_path.exists():
|
||||
raise SystemExit(f"找不到 Style: {style_path}")
|
||||
|
||||
targets = set(args.source_layers or DEFAULT_TARGETS)
|
||||
style = json.loads(style_path.read_text(encoding="utf-8"))
|
||||
|
||||
style_by_source_layer: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for layer in style.get("layers", []):
|
||||
if layer.get("source") == "navsea_delivery" and layer.get("source-layer"):
|
||||
style_by_source_layer[str(layer["source-layer"])].append(layer)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
counts = Counter()
|
||||
type_counts = Counter()
|
||||
property_key_counts = Counter()
|
||||
|
||||
for path in sorted(pbf_root.rglob("*.pbf")):
|
||||
rel = path.relative_to(pbf_root)
|
||||
if len(rel.parts) < 3:
|
||||
continue
|
||||
z = int(rel.parts[-3])
|
||||
x = int(rel.parts[-2])
|
||||
y = int(Path(rel.parts[-1]).stem)
|
||||
|
||||
decoded = mapbox_vector_tile.decode(path.read_bytes())
|
||||
for source_layer, layer_obj in decoded.items():
|
||||
if source_layer not in targets:
|
||||
continue
|
||||
|
||||
candidates = style_by_source_layer.get(source_layer, [])
|
||||
for feature in layer_obj.get("features", []):
|
||||
props = feature.get("properties") or {}
|
||||
filter_results = [
|
||||
compact_filter_result(sl, props)
|
||||
for sl in candidates
|
||||
]
|
||||
if any(r["filter_matched"] for r in filter_results):
|
||||
continue
|
||||
|
||||
output_id = feature.get("id")
|
||||
geom = feature.get("geometry") or {}
|
||||
geom_type = geom.get("type") if isinstance(geom, dict) else None
|
||||
|
||||
classification = classify_missing(source_layer, props)
|
||||
row = {
|
||||
"z": z,
|
||||
"x": x,
|
||||
"y": y,
|
||||
"tile": f"{z}/{x}/{y}",
|
||||
"source_layer": source_layer,
|
||||
"output_id": output_id,
|
||||
"geometry_type": geom_type,
|
||||
"classification": classification,
|
||||
"properties": dict(sorted(props.items())),
|
||||
"style_filter_results": filter_results,
|
||||
"pbf_path": str(path),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
counts[(z, source_layer)] += 1
|
||||
type_counts[
|
||||
(
|
||||
source_layer,
|
||||
str(props.get("canonical_object_type") or "<missing>"),
|
||||
)
|
||||
] += 1
|
||||
for key in props:
|
||||
property_key_counts[(source_layer, key)] += 1
|
||||
|
||||
result = {
|
||||
"schema": "navsea.cm95.style-contract-unmatched-features/v1",
|
||||
"release_dir": str(release_dir),
|
||||
"style_path": str(style_path),
|
||||
"profile": args.profile,
|
||||
"target_source_layers": sorted(targets),
|
||||
"unmatched_count": len(rows),
|
||||
"counts_by_zoom_source_layer": [
|
||||
{"z": z, "source_layer": sl, "count": count}
|
||||
for (z, sl), count in sorted(counts.items())
|
||||
],
|
||||
"canonical_object_type_counts": [
|
||||
{
|
||||
"source_layer": sl,
|
||||
"canonical_object_type": obj_type,
|
||||
"count": count,
|
||||
}
|
||||
for (sl, obj_type), count in sorted(
|
||||
type_counts.items(),
|
||||
key=lambda kv: (kv[0][0], -kv[1], kv[0][1]),
|
||||
)
|
||||
],
|
||||
"property_key_counts": [
|
||||
{"source_layer": sl, "property": key, "count": count}
|
||||
for (sl, key), count in sorted(
|
||||
property_key_counts.items(),
|
||||
key=lambda kv: (kv[0][0], -kv[1], kv[0][1]),
|
||||
)
|
||||
],
|
||||
"features": rows,
|
||||
}
|
||||
|
||||
output_path.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2, default=str),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
txt_path = output_path.with_suffix(".txt")
|
||||
lines = [
|
||||
"CM95 Style Contract unmatched Feature 精确提取器 v1",
|
||||
"=" * 76,
|
||||
f"Release : {release_dir}",
|
||||
f"Style : {style_path}",
|
||||
f"Unmatched : {len(rows)}",
|
||||
"",
|
||||
"Canonical object types:",
|
||||
]
|
||||
for item in result["canonical_object_type_counts"]:
|
||||
lines.append(
|
||||
f" {item['source_layer']:<24} "
|
||||
f"{item['canonical_object_type']:<28} "
|
||||
f"{item['count']}"
|
||||
)
|
||||
|
||||
lines += ["", "Exact unmatched features:"]
|
||||
for row in rows:
|
||||
c = row["classification"]
|
||||
lines.append(
|
||||
f" {row['tile']:<14} "
|
||||
f"{row['source_layer']:<24} "
|
||||
f"id={str(row['output_id']):<12} "
|
||||
f"type={str(c.get('canonical_object_type')):<24} "
|
||||
f"issue={c.get('likely_issue')}"
|
||||
)
|
||||
lines.append(
|
||||
" props="
|
||||
+ json.dumps(
|
||||
row["properties"],
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
)
|
||||
|
||||
txt_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
print("\n".join(lines))
|
||||
print("")
|
||||
print("JSON:", output_path)
|
||||
print("TXT :", txt_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user