docs(pbf): archive Full production provenance and engineering history
This commit is contained in:
OpenAI Codex
14
tools/visual-object-instantiation/README.md
Normal file
14
tools/visual-object-instantiation/README.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# Visual Object Instantiation
|
||||
|
||||
该工具是 Style Visual Object Tool 的下游阶段。
|
||||
|
||||
职责：读取已经完成 Style 解析的视觉规则，并用真实 PBF Feature 的实际属性把动态表达式实例化为“实际出现过的 Anonymous Visual Object”。
|
||||
|
||||
边界：
|
||||
- 不给 Layer 命名；
|
||||
- 不解释 provider 字段语义；
|
||||
- source-layer 只用于选择对应 Style Layer，并在输出中作为 provenance；
|
||||
- 不把 source-layer 名称作为 Semantic；
|
||||
- 不修改 Style Visual Object Tool 的 Provider-neutral 规则。
|
||||
|
||||
流程：Style Definition Catalog + PBF -> Actual Anonymous Visual Object Catalog。
|
||||
@@ -0,0 +1,491 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from mapbox_vector_tile.Mapbox import vector_tile_pb2
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def canonical(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def stable_id(prefix: str, value: Any) -> str:
|
||||
return prefix + "-" + hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
|
||||
def protobuf_value(value: Any) -> Any:
|
||||
fields = value.ListFields()
|
||||
if not fields:
|
||||
return None
|
||||
return fields[0][1]
|
||||
|
||||
|
||||
def decode_feature_properties(layer: Any, feature: Any) -> dict[str, Any]:
|
||||
props: dict[str, Any] = {}
|
||||
tags = feature.tags
|
||||
for i in range(0, len(tags), 2):
|
||||
if i + 1 >= len(tags):
|
||||
break
|
||||
key_index = tags[i]
|
||||
value_index = tags[i + 1]
|
||||
if key_index >= len(layer.keys) or value_index >= len(layer.values):
|
||||
continue
|
||||
props[layer.keys[key_index]] = protobuf_value(layer.values[value_index])
|
||||
return props
|
||||
|
||||
|
||||
def geometry_role(feature_type: int) -> str:
|
||||
return {1: "Point", 2: "LineString", 3: "Polygon"}.get(int(feature_type), "Unknown")
|
||||
|
||||
def numeric(value: Any) -> float | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def evaluate(expr: Any, props: dict[str, Any], zoom: int) -> Any:
|
||||
if not isinstance(expr, list):
|
||||
return expr
|
||||
if not expr:
|
||||
return expr
|
||||
head = expr[0]
|
||||
if not isinstance(head, str):
|
||||
return [evaluate(x, props, zoom) for x in expr]
|
||||
|
||||
if head == "get":
|
||||
return props.get(expr[1]) if len(expr) > 1 else None
|
||||
if head == "has":
|
||||
if len(expr) < 2:
|
||||
return False
|
||||
key = expr[1]
|
||||
return key in props and props.get(key) not in (None, "")
|
||||
if head == "!":
|
||||
return not bool(evaluate(expr[1], props, zoom)) if len(expr) > 1 else True
|
||||
if head == "all":
|
||||
return all(bool(evaluate(x, props, zoom)) for x in expr[1:])
|
||||
if head == "any":
|
||||
return any(bool(evaluate(x, props, zoom)) for x in expr[1:])
|
||||
if head in {"==", "!=", ">", "<", ">=", "<="}:
|
||||
if len(expr) < 3:
|
||||
return False
|
||||
a = evaluate(expr[1], props, zoom)
|
||||
b = evaluate(expr[2], props, zoom)
|
||||
if head == "==":
|
||||
return a == b
|
||||
if head == "!=":
|
||||
return a != b
|
||||
an = numeric(a)
|
||||
bn = numeric(b)
|
||||
if an is not None and bn is not None:
|
||||
a, b = an, bn
|
||||
try:
|
||||
return {
|
||||
">": a > b,
|
||||
"<": a < b,
|
||||
">=": a >= b,
|
||||
"<=": a <= b,
|
||||
}[head]
|
||||
except TypeError:
|
||||
return False
|
||||
if head == "literal":
|
||||
return expr[1] if len(expr) > 1 else None
|
||||
if head == "to-number":
|
||||
value = evaluate(expr[1], props, zoom) if len(expr) > 1 else None
|
||||
n = numeric(value)
|
||||
if n is not None:
|
||||
return n
|
||||
return evaluate(expr[2], props, zoom) if len(expr) > 2 else None
|
||||
if head == "to-string":
|
||||
value = evaluate(expr[1], props, zoom) if len(expr) > 1 else None
|
||||
return "" if value is None else str(value)
|
||||
if head == "concat":
|
||||
parts = []
|
||||
for item in expr[1:]:
|
||||
value = evaluate(item, props, zoom)
|
||||
parts.append("" if value is None else str(value))
|
||||
return "".join(parts)
|
||||
if head == "coalesce":
|
||||
for item in expr[1:]:
|
||||
value = evaluate(item, props, zoom)
|
||||
if value not in (None, ""):
|
||||
return value
|
||||
return None
|
||||
if head == "case":
|
||||
clauses = expr[1:]
|
||||
for i in range(0, len(clauses) - 1, 2):
|
||||
if bool(evaluate(clauses[i], props, zoom)):
|
||||
return evaluate(clauses[i + 1], props, zoom)
|
||||
return evaluate(clauses[-1], props, zoom) if clauses else None
|
||||
if head == "match":
|
||||
if len(expr) < 4:
|
||||
return None
|
||||
value = evaluate(expr[1], props, zoom)
|
||||
arms = expr[2:]
|
||||
fallback = arms[-1]
|
||||
pairs = arms[:-1]
|
||||
for i in range(0, len(pairs), 2):
|
||||
if i + 1 >= len(pairs):
|
||||
break
|
||||
labels, result = pairs[i], pairs[i + 1]
|
||||
if isinstance(labels, list):
|
||||
matched = value in labels
|
||||
else:
|
||||
matched = value == labels
|
||||
if matched:
|
||||
return evaluate(result, props, zoom)
|
||||
return evaluate(fallback, props, zoom)
|
||||
if head == "zoom":
|
||||
return zoom
|
||||
if head == "step":
|
||||
if len(expr) < 3:
|
||||
return None
|
||||
input_value = numeric(evaluate(expr[1], props, zoom))
|
||||
result = evaluate(expr[2], props, zoom)
|
||||
if input_value is None:
|
||||
return result
|
||||
stops = expr[3:]
|
||||
for i in range(0, len(stops), 2):
|
||||
if i + 1 >= len(stops):
|
||||
break
|
||||
stop = numeric(evaluate(stops[i], props, zoom))
|
||||
if stop is None:
|
||||
continue
|
||||
if input_value >= stop:
|
||||
result = evaluate(stops[i + 1], props, zoom)
|
||||
else:
|
||||
break
|
||||
return result
|
||||
|
||||
# Unknown operators stay explicit instead of silently becoming truthy.
|
||||
return {"unsupported_expression": expr}
|
||||
|
||||
|
||||
def visible(layer: dict[str, Any], zoom: int) -> bool:
|
||||
if zoom < float(layer.get("minzoom", 0)):
|
||||
return False
|
||||
maxzoom = layer.get("maxzoom")
|
||||
if maxzoom is not None and zoom >= float(maxzoom):
|
||||
return False
|
||||
if (layer.get("layout") or {}).get("visibility") == "none":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def matches(layer: dict[str, Any], props: dict[str, Any], zoom: int) -> bool:
|
||||
if not visible(layer, zoom):
|
||||
return False
|
||||
filt = layer.get("filter")
|
||||
if filt is None:
|
||||
return True
|
||||
value = evaluate(filt, props, zoom)
|
||||
return value is True
|
||||
|
||||
|
||||
def clean_value(value: Any) -> Any:
|
||||
if isinstance(value, float) and math.isfinite(value):
|
||||
return round(value, 6)
|
||||
return value
|
||||
|
||||
|
||||
def sprite_hashes(meta: dict[str, Any], atlas: Image.Image) -> dict[str, str]:
|
||||
out = {}
|
||||
for key, m in meta.items():
|
||||
crop = atlas.crop((
|
||||
int(m["x"]), int(m["y"]),
|
||||
int(m["x"]) + int(m["width"]),
|
||||
int(m["y"]) + int(m["height"]),
|
||||
))
|
||||
out[key] = hashlib.sha256(crop.tobytes()).hexdigest()
|
||||
return out
|
||||
|
||||
|
||||
def component_from_layer(
|
||||
layer: dict[str, Any],
|
||||
style_index: int,
|
||||
props: dict[str, Any],
|
||||
zoom: int,
|
||||
sprite_meta: dict[str, Any],
|
||||
visual_hashes: dict[str, str],
|
||||
) -> list[dict[str, Any]]:
|
||||
ltype = layer.get("type")
|
||||
layout = layer.get("layout") or {}
|
||||
paint = layer.get("paint") or {}
|
||||
out: list[dict[str, Any]] = []
|
||||
|
||||
if ltype == "symbol":
|
||||
if "icon-image" in layout:
|
||||
key = evaluate(layout["icon-image"], props, zoom)
|
||||
if isinstance(key, str) and key:
|
||||
out.append({
|
||||
"style_index": style_index,
|
||||
"style_layer_id": layer.get("id"),
|
||||
"role": "icon",
|
||||
"resource_key": key,
|
||||
"resource_visual_hash": visual_hashes.get(key),
|
||||
"resource_present": key in sprite_meta,
|
||||
"icon_size": clean_value(evaluate(layout.get("icon-size", 1.0), props, zoom)),
|
||||
"icon_rotate": clean_value(evaluate(layout.get("icon-rotate", 0.0), props, zoom)),
|
||||
"icon_anchor": clean_value(evaluate(layout.get("icon-anchor", "center"), props, zoom)),
|
||||
})
|
||||
if "text-field" in layout:
|
||||
text_value = evaluate(layout["text-field"], props, zoom)
|
||||
if text_value not in (None, ""):
|
||||
out.append({
|
||||
"style_index": style_index,
|
||||
"style_layer_id": layer.get("id"),
|
||||
"role": "text",
|
||||
"text_present": True,
|
||||
"text_size": clean_value(evaluate(layout.get("text-size"), props, zoom)),
|
||||
"text_anchor": clean_value(evaluate(layout.get("text-anchor"), props, zoom)),
|
||||
"text_offset": clean_value(evaluate(layout.get("text-offset"), props, zoom)),
|
||||
"text_color": clean_value(evaluate(paint.get("text-color"), props, zoom)),
|
||||
"text_halo_color": clean_value(evaluate(paint.get("text-halo-color"), props, zoom)),
|
||||
"text_halo_width": clean_value(evaluate(paint.get("text-halo-width"), props, zoom)),
|
||||
})
|
||||
|
||||
elif ltype == "line":
|
||||
key = evaluate(paint.get("line-pattern"), props, zoom) if "line-pattern" in paint else None
|
||||
out.append({
|
||||
"style_index": style_index,
|
||||
"style_layer_id": layer.get("id"),
|
||||
"role": "line",
|
||||
"resource_key": key if isinstance(key, str) and key else None,
|
||||
"resource_visual_hash": visual_hashes.get(key) if isinstance(key, str) else None,
|
||||
"resource_present": key in sprite_meta if isinstance(key, str) else None,
|
||||
"line_color": clean_value(evaluate(paint.get("line-color"), props, zoom)),
|
||||
"line_width": clean_value(evaluate(paint.get("line-width"), props, zoom)),
|
||||
"line_dasharray": clean_value(evaluate(paint.get("line-dasharray"), props, zoom)),
|
||||
"line_opacity": clean_value(evaluate(paint.get("line-opacity"), props, zoom)),
|
||||
})
|
||||
|
||||
elif ltype == "fill":
|
||||
key = evaluate(paint.get("fill-pattern"), props, zoom) if "fill-pattern" in paint else None
|
||||
out.append({
|
||||
"style_index": style_index,
|
||||
"style_layer_id": layer.get("id"),
|
||||
"role": "fill",
|
||||
"resource_key": key if isinstance(key, str) and key else None,
|
||||
"resource_visual_hash": visual_hashes.get(key) if isinstance(key, str) else None,
|
||||
"resource_present": key in sprite_meta if isinstance(key, str) else None,
|
||||
"fill_color": clean_value(evaluate(paint.get("fill-color"), props, zoom)),
|
||||
"fill_outline_color": clean_value(evaluate(paint.get("fill-outline-color"), props, zoom)),
|
||||
"fill_opacity": clean_value(evaluate(paint.get("fill-opacity"), props, zoom)),
|
||||
})
|
||||
|
||||
elif ltype == "circle":
|
||||
out.append({
|
||||
"style_index": style_index,
|
||||
"style_layer_id": layer.get("id"),
|
||||
"role": "circle",
|
||||
"circle_color": clean_value(evaluate(paint.get("circle-color"), props, zoom)),
|
||||
"circle_radius": clean_value(evaluate(paint.get("circle-radius"), props, zoom)),
|
||||
"circle_stroke_color": clean_value(evaluate(paint.get("circle-stroke-color"), props, zoom)),
|
||||
"circle_stroke_width": clean_value(evaluate(paint.get("circle-stroke-width"), props, zoom)),
|
||||
})
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def signature_components(components: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
# Exclude provider/resource names and Style layer IDs from the visual signature.
|
||||
result = []
|
||||
for c in components:
|
||||
x = {k: v for k, v in c.items() if k not in {"style_layer_id", "resource_key"}}
|
||||
result.append(x)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--style", required=True)
|
||||
ap.add_argument("--sprite-json", required=True)
|
||||
ap.add_argument("--sprite-png", required=True)
|
||||
ap.add_argument("--pbf-root", required=True)
|
||||
ap.add_argument("--output", required=True)
|
||||
ap.add_argument("--progress-every", type=int, default=5000)
|
||||
args = ap.parse_args()
|
||||
|
||||
style = json.loads(Path(args.style).read_text(encoding="utf-8"))
|
||||
sprite_meta = json.loads(Path(args.sprite_json).read_text(encoding="utf-8"))
|
||||
atlas = Image.open(args.sprite_png).convert("RGBA")
|
||||
visual_hashes = sprite_hashes(sprite_meta, atlas)
|
||||
|
||||
indexed: dict[str, list[tuple[int, dict[str, Any]]]] = defaultdict(list)
|
||||
for i, layer in enumerate(style.get("layers", [])):
|
||||
sl = layer.get("source-layer")
|
||||
if sl:
|
||||
indexed[sl].append((i, layer))
|
||||
|
||||
root = Path(args.pbf_root)
|
||||
paths = sorted(root.glob("*/*/*.pbf"))
|
||||
output = Path(args.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
distinct: dict[str, dict[str, Any]] = {}
|
||||
group_stats: dict[str, Counter] = defaultdict(Counter)
|
||||
source_group_map: dict[str, str] = {}
|
||||
layer_seen = Counter()
|
||||
feature_count = 0
|
||||
matched_feature_count = 0
|
||||
unsupported_count = 0
|
||||
missing_sprite_count = 0
|
||||
started = time.time()
|
||||
|
||||
for tile_idx, path in enumerate(paths, 1):
|
||||
parts = path.relative_to(root).parts
|
||||
z, x, yfile = int(parts[0]), int(parts[1]), parts[2]
|
||||
y = int(Path(yfile).stem)
|
||||
try:
|
||||
decoded = vector_tile_pb2.tile()
|
||||
decoded.ParseFromString(path.read_bytes())
|
||||
except Exception as exc:
|
||||
print(f"DECODE_FAIL {path}: {exc}", flush=True)
|
||||
continue
|
||||
|
||||
for tile_layer in decoded.layers:
|
||||
layer_name = tile_layer.name
|
||||
layer_seen[layer_name] += len(tile_layer.features)
|
||||
style_layers = indexed.get(layer_name, [])
|
||||
if not style_layers:
|
||||
continue
|
||||
group_id = source_group_map.setdefault(
|
||||
layer_name,
|
||||
stable_id("GRP", {"source": "cm95", "source_layer": layer_name}),
|
||||
)
|
||||
for feature in tile_layer.features:
|
||||
feature_count += 1
|
||||
props = decode_feature_properties(tile_layer, feature)
|
||||
components: list[dict[str, Any]] = []
|
||||
for style_index, style_layer in style_layers:
|
||||
if not matches(style_layer, props, z):
|
||||
continue
|
||||
comps = component_from_layer(
|
||||
style_layer, style_index, props, z, sprite_meta, visual_hashes
|
||||
)
|
||||
for comp in comps:
|
||||
if isinstance(comp, dict):
|
||||
if any(
|
||||
isinstance(v, dict) and "unsupported_expression" in v
|
||||
for v in comp.values()
|
||||
):
|
||||
unsupported_count += 1
|
||||
if comp.get("resource_key") and comp.get("resource_present") is False:
|
||||
missing_sprite_count += 1
|
||||
components.extend(comps)
|
||||
|
||||
if not components:
|
||||
continue
|
||||
matched_feature_count += 1
|
||||
visual_signature = {
|
||||
"geometry_role": geometry_role(feature.type),
|
||||
"components": signature_components(components),
|
||||
}
|
||||
vo_id = stable_id("VO", visual_signature)
|
||||
item = distinct.get(vo_id)
|
||||
if item is None:
|
||||
item = {
|
||||
"visual_object_id": vo_id,
|
||||
"group_id": group_id,
|
||||
"geometry_role": visual_signature["geometry_role"],
|
||||
"components": components,
|
||||
"occurrence_count": 0,
|
||||
"provenance": {
|
||||
"source_layer": layer_name,
|
||||
},
|
||||
"example": {
|
||||
"tile": f"{z}/{x}/{y}",
|
||||
"feature_id": int(feature.id) if feature.HasField("id") else None,
|
||||
},
|
||||
}
|
||||
distinct[vo_id] = item
|
||||
item["occurrence_count"] += 1
|
||||
group_stats[group_id]["tile_feature_occurrences"] += 1
|
||||
group_stats[group_id]["distinct_vo_ids"] = 0
|
||||
|
||||
if tile_idx % args.progress_every == 0 or tile_idx == len(paths):
|
||||
elapsed = time.time() - started
|
||||
print(
|
||||
f"PROGRESS tiles={tile_idx}/{len(paths)} features={feature_count} "
|
||||
f"distinct_vo={len(distinct)} elapsed={elapsed:.1f}s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Count distinct VOs per group after the scan.
|
||||
per_group_distinct = Counter(item["group_id"] for item in distinct.values())
|
||||
groups = []
|
||||
for source_layer, group_id in sorted(source_group_map.items()):
|
||||
groups.append({
|
||||
"group_id": group_id,
|
||||
"distinct_visual_object_count": per_group_distinct[group_id],
|
||||
"tile_feature_occurrences": group_stats[group_id]["tile_feature_occurrences"],
|
||||
"provenance": {"source_layer": source_layer},
|
||||
})
|
||||
|
||||
catalog = {
|
||||
"schema": "actual-visual-object-catalog/v1",
|
||||
"catalog_name": "cm95-actual-anonymous",
|
||||
"principle": (
|
||||
"PBF is used only to instantiate Style visual expressions and occurrence combinations; "
|
||||
"provider semantic names are not used for classification."
|
||||
),
|
||||
"inputs": {
|
||||
"style": str(Path(args.style)),
|
||||
"sprite_json": str(Path(args.sprite_json)),
|
||||
"sprite_png": str(Path(args.sprite_png)),
|
||||
"pbf_root": str(root),
|
||||
"tile_count": len(paths),
|
||||
},
|
||||
"summary": {
|
||||
"decoded_feature_count_in_styled_layers": feature_count,
|
||||
"matched_feature_occurrence_count": matched_feature_count,
|
||||
"distinct_visual_object_count": len(distinct),
|
||||
"group_count": len(groups),
|
||||
"unsupported_component_value_count": unsupported_count,
|
||||
"missing_sprite_component_count": missing_sprite_count,
|
||||
"source_layer_occurrence_counts": dict(layer_seen),
|
||||
},
|
||||
"groups": groups,
|
||||
"visual_objects": sorted(distinct.values(), key=lambda x: x["visual_object_id"]),
|
||||
}
|
||||
(output / "actual-anonymous-catalog.json").write_text(
|
||||
json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
sanitized = {
|
||||
"schema": catalog["schema"],
|
||||
"catalog_name": catalog["catalog_name"],
|
||||
"summary": catalog["summary"],
|
||||
"groups": [
|
||||
{k: v for k, v in g.items() if k != "provenance"}
|
||||
for g in catalog["groups"]
|
||||
],
|
||||
"visual_objects": [
|
||||
{k: v for k, v in obj.items() if k not in {"provenance", "example"}}
|
||||
for obj in catalog["visual_objects"]
|
||||
],
|
||||
}
|
||||
(output / "matcher-input-sanitized.json").write_text(
|
||||
json.dumps(sanitized, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(json.dumps(catalog["summary"], ensure_ascii=False, indent=2), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
11
tools/visual-object-matcher/README.md
Normal file
11
tools/visual-object-matcher/README.md
Normal file
@@ -0,0 +1,11 @@
|
||||
# Visual Object Matcher
|
||||
|
||||
输入：
|
||||
1. 已冻结的 NavSea Standard Visual Object Catalog；
|
||||
2. Provider 的 Actual Anonymous Visual Object Catalog。
|
||||
|
||||
匹配依据只使用视觉结构：Sprite 图像、组件角色、组件顺序、几何角色、line/fill/pattern/text role 等。
|
||||
|
||||
禁止使用 Provider source-layer 名称、Provider 名称、对象类名称或字段名称直接决定标准 Layer。
|
||||
|
||||
Provider source-layer 仅允许在匹配完成后用于 blind validation / audit，不能进入 matcher input。
|
||||
358
tools/visual-object-matcher/visual_matcher_experimental.py
Normal file
358
tools/visual-object-matcher/visual_matcher_experimental.py
Normal file
@@ -0,0 +1,358 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageFilter
|
||||
|
||||
|
||||
def load_catalog(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_atlas(catalog_dir: Path) -> Image.Image:
|
||||
return Image.open(catalog_dir / "_sprite" / "sprite.png").convert("RGBA")
|
||||
|
||||
|
||||
def visual_feature(atlas: Image.Image, meta: dict[str, Any]) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
crop = atlas.crop((
|
||||
int(meta["x"]), int(meta["y"]),
|
||||
int(meta["x"]) + int(meta["width"]),
|
||||
int(meta["y"]) + int(meta["height"]),
|
||||
))
|
||||
alpha = np.asarray(crop.getchannel("A"))
|
||||
ys, xs = np.where(alpha > 8)
|
||||
if len(xs):
|
||||
crop = crop.crop((int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1))
|
||||
|
||||
w, h = crop.size
|
||||
scale = min(54.0 / max(w, 1), 54.0 / max(h, 1))
|
||||
nw = max(1, round(w * scale))
|
||||
nh = max(1, round(h * scale))
|
||||
crop = crop.resize((nw, nh), Image.Resampling.LANCZOS)
|
||||
|
||||
canvas = Image.new("RGBA", (64, 64), (255, 255, 255, 0))
|
||||
canvas.alpha_composite(crop, ((64 - nw) // 2, (64 - nh) // 2))
|
||||
arr = np.asarray(canvas, dtype=np.float32) / 255.0
|
||||
a = arr[:, :, 3]
|
||||
rgb = arr[:, :, :3] * a[:, :, None] + (1.0 - a[:, :, None])
|
||||
edge = np.asarray(canvas.convert("L").filter(ImageFilter.FIND_EDGES), dtype=np.float32) / 255.0
|
||||
return rgb, a, edge
|
||||
|
||||
|
||||
def cosine(a: np.ndarray, b: np.ndarray) -> float:
|
||||
aa = a.ravel()
|
||||
bb = b.ravel()
|
||||
denom = float(np.linalg.norm(aa) * np.linalg.norm(bb))
|
||||
return float(np.dot(aa, bb) / denom) if denom else 0.0
|
||||
|
||||
|
||||
def image_similarity(a: tuple[np.ndarray, np.ndarray, np.ndarray], b: tuple[np.ndarray, np.ndarray, np.ndarray]) -> float:
|
||||
rgb1, alpha1, edge1 = a
|
||||
rgb2, alpha2, edge2 = b
|
||||
rgb_score = 1.0 - float(np.mean((rgb1 - rgb2) ** 2))
|
||||
alpha_score = cosine(alpha1, alpha2)
|
||||
edge_score = cosine(edge1, edge2)
|
||||
return 0.45 * rgb_score + 0.35 * alpha_score + 0.20 * edge_score
|
||||
|
||||
|
||||
def unique_sprite_objects(catalog: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
seen: set[tuple[str, str, str | None]] = set()
|
||||
out: list[dict[str, Any]] = []
|
||||
for obj in catalog.get("visual_objects", []):
|
||||
if not obj.get("resource_key") or not obj.get("resource_meta"):
|
||||
continue
|
||||
key = (
|
||||
obj.get("component_role") or "",
|
||||
obj.get("visual_hash") or "",
|
||||
obj.get("standard_layer_name"),
|
||||
obj.get("group_id"),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(obj)
|
||||
return out
|
||||
|
||||
|
||||
def build_features(catalog: dict[str, Any], catalog_dir: Path) -> tuple[list[dict[str, Any]], list[tuple[np.ndarray, np.ndarray, np.ndarray]]]:
|
||||
atlas = load_atlas(catalog_dir)
|
||||
objects = unique_sprite_objects(catalog)
|
||||
features = [visual_feature(atlas, obj["resource_meta"]) for obj in objects]
|
||||
return objects, features
|
||||
|
||||
|
||||
def compatible(cm: dict[str, Any], std: dict[str, Any]) -> bool:
|
||||
if cm.get("component_role") != std.get("component_role"):
|
||||
return False
|
||||
cg = cm.get("geometry_role")
|
||||
sg = std.get("geometry_role")
|
||||
if cg and sg and cg != sg:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def classify_object(
|
||||
cm_obj: dict[str, Any],
|
||||
cm_feat: tuple[np.ndarray, np.ndarray, np.ndarray],
|
||||
std_objects: list[dict[str, Any]],
|
||||
std_features: list[tuple[np.ndarray, np.ndarray, np.ndarray]],
|
||||
) -> dict[str, Any]:
|
||||
layer_best: dict[str, tuple[float, str]] = {}
|
||||
for std, feat in zip(std_objects, std_features):
|
||||
layer = std.get("standard_layer_name")
|
||||
if not layer or not compatible(cm_obj, std):
|
||||
continue
|
||||
score = image_similarity(cm_feat, feat)
|
||||
prev = layer_best.get(layer)
|
||||
if prev is None or score > prev[0]:
|
||||
layer_best[layer] = (score, std["visual_object_id"])
|
||||
|
||||
ranked = sorted(
|
||||
(
|
||||
{"layer": layer, "score": score, "standard_visual_object_id": vo_id}
|
||||
for layer, (score, vo_id) in layer_best.items()
|
||||
),
|
||||
key=lambda x: x["score"],
|
||||
reverse=True,
|
||||
)
|
||||
top = ranked[0] if ranked else None
|
||||
second = ranked[1] if len(ranked) > 1 else None
|
||||
if not top:
|
||||
status = "UNKNOWN"
|
||||
else:
|
||||
margin = top["score"] - (second["score"] if second else 0.0)
|
||||
if top["score"] >= 0.90 and margin >= 0.025:
|
||||
status = "MATCH"
|
||||
elif top["score"] >= 0.82:
|
||||
status = "PARTIAL"
|
||||
else:
|
||||
status = "UNKNOWN"
|
||||
return {
|
||||
"visual_object_id": cm_obj["visual_object_id"],
|
||||
"group_id": cm_obj["group_id"],
|
||||
"component_role": cm_obj.get("component_role"),
|
||||
"geometry_role": cm_obj.get("geometry_role"),
|
||||
"resource_key": cm_obj.get("resource_key"),
|
||||
"status": status,
|
||||
"best": top,
|
||||
"alternatives": ranked[1:4],
|
||||
}
|
||||
|
||||
|
||||
def group_scores(object_matches: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for item in object_matches:
|
||||
grouped[item["group_id"]].append(item)
|
||||
|
||||
results: dict[str, dict[str, Any]] = {}
|
||||
for group_id, items in grouped.items():
|
||||
evidence: dict[str, list[float]] = defaultdict(list)
|
||||
for item in items:
|
||||
for cand in ([item["best"]] if item.get("best") else []) + item.get("alternatives", []):
|
||||
if cand and cand["score"] >= 0.78:
|
||||
evidence[cand["layer"]].append(cand["score"])
|
||||
ranking = []
|
||||
for layer, scores in evidence.items():
|
||||
scores = sorted(scores, reverse=True)
|
||||
strongest = scores[: min(12, len(scores))]
|
||||
coverage = min(1.0, len(scores) / max(3.0, len(items) * 0.25))
|
||||
score = (sum(strongest) / len(strongest)) * (0.65 + 0.35 * coverage)
|
||||
ranking.append({
|
||||
"layer": layer,
|
||||
"score": score,
|
||||
"evidence_count": len(scores),
|
||||
"best_evidence": scores[:5],
|
||||
})
|
||||
ranking.sort(key=lambda x: x["score"], reverse=True)
|
||||
top = ranking[0] if ranking else None
|
||||
second = ranking[1] if len(ranking) > 1 else None
|
||||
if top:
|
||||
margin = top["score"] - (second["score"] if second else 0.0)
|
||||
dominance = top["evidence_count"] / max(1, sum(x["evidence_count"] for x in ranking))
|
||||
if top["score"] >= 0.86 and margin >= 0.035 and dominance >= 0.40:
|
||||
status = "MATCH"
|
||||
elif top["score"] >= 0.80:
|
||||
status = "PARTIAL"
|
||||
else:
|
||||
status = "UNKNOWN"
|
||||
else:
|
||||
status = "UNKNOWN"
|
||||
results[group_id] = {
|
||||
"group_id": group_id,
|
||||
"status": status,
|
||||
"best": top,
|
||||
"alternatives": ranking[1:4],
|
||||
"object_count": len(items),
|
||||
}
|
||||
return results
|
||||
|
||||
|
||||
def apply_group_result(object_matches: list[dict[str, Any]], groups: dict[str, dict[str, Any]]) -> None:
|
||||
for item in object_matches:
|
||||
g = groups.get(item["group_id"])
|
||||
if g and g["status"] == "MATCH":
|
||||
item["final_layer"] = g["best"]["layer"]
|
||||
item["final_basis"] = "GROUP_VISUAL_CONSENSUS"
|
||||
item["final_status"] = "MATCH"
|
||||
elif item["status"] == "MATCH":
|
||||
item["final_layer"] = item["best"]["layer"]
|
||||
item["final_basis"] = "OBJECT_VISUAL_MATCH"
|
||||
item["final_status"] = "MATCH"
|
||||
elif item["status"] == "PARTIAL":
|
||||
item["final_layer"] = item["best"]["layer"] if item.get("best") else None
|
||||
item["final_basis"] = "OBJECT_VISUAL_PARTIAL"
|
||||
item["final_status"] = "PARTIAL"
|
||||
else:
|
||||
item["final_layer"] = None
|
||||
item["final_basis"] = "INSUFFICIENT_VISUAL_EVIDENCE"
|
||||
item["final_status"] = "UNKNOWN"
|
||||
|
||||
|
||||
def blind_validation(
|
||||
cm_catalog: dict[str, Any],
|
||||
group_results: dict[str, dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
# This is deliberately post-match audit only. Raw source-layer names are read
|
||||
# after predictions have already been produced.
|
||||
provenance = {g["group_id"]: g.get("provenance", {}) for g in cm_catalog.get("groups", [])}
|
||||
rows = []
|
||||
comparable = correct = 0
|
||||
for group_id, pred in sorted(group_results.items()):
|
||||
raw_name = provenance.get(group_id, {}).get("source_layer")
|
||||
predicted = pred.get("best", {}).get("layer") if pred.get("best") else None
|
||||
row = {
|
||||
"group_id": group_id,
|
||||
"raw_source_layer_for_audit_only": raw_name,
|
||||
"predicted_standard_layer": predicted,
|
||||
"status": pred.get("status"),
|
||||
"same_name_after_blind_match": bool(raw_name and predicted and raw_name == predicted),
|
||||
}
|
||||
if raw_name and predicted and not str(raw_name).startswith("s52_"):
|
||||
comparable += 1
|
||||
if raw_name == predicted:
|
||||
correct += 1
|
||||
rows.append(row)
|
||||
return {
|
||||
"note": "Raw CM95 source-layer is used only after matching for blind validation; it is not matcher input.",
|
||||
"comparable_group_count": comparable,
|
||||
"same_name_count": correct,
|
||||
"same_name_rate": (correct / comparable) if comparable else None,
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
|
||||
def html_report(result: dict[str, Any]) -> str:
|
||||
stats = result["summary"]
|
||||
val = result["blind_validation"]
|
||||
rows = []
|
||||
for g in result["group_matches"]:
|
||||
best = g.get("best") or {}
|
||||
alts = ", ".join(f'{x["layer"]} {x["score"]:.3f}' for x in g.get("alternatives", []))
|
||||
rows.append(
|
||||
"<tr>"
|
||||
f"<td><code>{g['group_id']}</code></td>"
|
||||
f"<td>{g['status']}</td>"
|
||||
f"<td>{best.get('layer','')}</td>"
|
||||
f"<td>{best.get('score',0):.3f}</td>"
|
||||
f"<td>{g['object_count']}</td>"
|
||||
f"<td>{alts}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
return f"""<!doctype html><meta charset='utf-8'><title>CM95 Visual Object Layer Match</title>
|
||||
<style>
|
||||
body{{font-family:Arial,sans-serif;margin:24px;color:#17202a}} table{{border-collapse:collapse;width:100%}}
|
||||
th,td{{border:1px solid #d8dee6;padding:7px;text-align:left;font-size:13px}} th{{background:#eef2f6}}
|
||||
code{{font-size:11px}} .box{{display:inline-block;margin:0 12px 12px 0;padding:10px 14px;border:1px solid #d8dee6;border-radius:8px}}
|
||||
</style>
|
||||
<h1>CM95 Visual Object → NavSea Standard Layer</h1>
|
||||
<p>匹配程序不读取 CM95 source-layer 名称；Layer 名仅来自 NavSea 已冻结 Standard Visual Object Catalog。</p>
|
||||
<div class='box'>CM95 Visual Objects <b>{stats['cm95_sprite_visual_objects']}</b></div>
|
||||
<div class='box'>MATCH <b>{stats['object_match']}</b></div>
|
||||
<div class='box'>PARTIAL <b>{stats['object_partial']}</b></div>
|
||||
<div class='box'>UNKNOWN <b>{stats['object_unknown']}</b></div>
|
||||
<div class='box'>Blind audit {val['same_name_count']}/{val['comparable_group_count']}</div>
|
||||
<h2>匿名 Group 的视觉匹配</h2>
|
||||
<table><thead><tr><th>匿名 Group</th><th>状态</th><th>预测标准 Layer</th><th>分数</th><th>VO 数</th><th>候选</th></tr></thead>
|
||||
<tbody>{''.join(rows)}</tbody></table>
|
||||
<p>详细到每个 Visual Object 的匹配、候选和 blind validation 见 <code>match-results.json</code>。</p>
|
||||
"""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--standard", required=True)
|
||||
ap.add_argument("--cm95", required=True)
|
||||
ap.add_argument("--output", required=True)
|
||||
args = ap.parse_args()
|
||||
|
||||
std_path = Path(args.standard)
|
||||
cm_path = Path(args.cm95)
|
||||
output = Path(args.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
std_catalog = load_catalog(std_path)
|
||||
cm_catalog = load_catalog(cm_path)
|
||||
std_objects, std_features = build_features(std_catalog, std_path.parent.parent / "navsea-anonymous")
|
||||
cm_objects, cm_features = build_features(cm_catalog, cm_path.parent)
|
||||
|
||||
# Sanitized match inputs deliberately omit CM95 provenance/source-layer.
|
||||
sanitized = [
|
||||
{
|
||||
"visual_object_id": o["visual_object_id"],
|
||||
"group_id": o["group_id"],
|
||||
"component_role": o.get("component_role"),
|
||||
"geometry_role": o.get("geometry_role"),
|
||||
"resource_key": o.get("resource_key"),
|
||||
}
|
||||
for o in cm_objects
|
||||
]
|
||||
(output / "cm95-match-input-sanitized.json").write_text(
|
||||
json.dumps(sanitized, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
object_matches = [
|
||||
classify_object(obj, feat, std_objects, std_features)
|
||||
for obj, feat in zip(cm_objects, cm_features)
|
||||
]
|
||||
groups = group_scores(object_matches)
|
||||
apply_group_result(object_matches, groups)
|
||||
validation = blind_validation(cm_catalog, groups)
|
||||
|
||||
result = {
|
||||
"schema": "style-visual-object-match/v1",
|
||||
"method": {
|
||||
"cm95_source_layer_used_for_matching": False,
|
||||
"provider_name_used_for_matching": False,
|
||||
"resource_key_text_semantics_used_for_matching": False,
|
||||
"visual_inputs": ["sprite pixels", "component role", "geometry role", "anonymous group membership"],
|
||||
"standard_labels_from": "frozen NavSea Standard Visual Object Catalog",
|
||||
},
|
||||
"summary": {
|
||||
"standard_sprite_visual_objects": len(std_objects),
|
||||
"cm95_sprite_visual_objects": len(cm_objects),
|
||||
"object_match": sum(x["final_status"] == "MATCH" for x in object_matches),
|
||||
"object_partial": sum(x["final_status"] == "PARTIAL" for x in object_matches),
|
||||
"object_unknown": sum(x["final_status"] == "UNKNOWN" for x in object_matches),
|
||||
"group_match": sum(x["status"] == "MATCH" for x in groups.values()),
|
||||
"group_partial": sum(x["status"] == "PARTIAL" for x in groups.values()),
|
||||
"group_unknown": sum(x["status"] == "UNKNOWN" for x in groups.values()),
|
||||
},
|
||||
"group_matches": sorted(groups.values(), key=lambda x: x["group_id"]),
|
||||
"visual_object_matches": object_matches,
|
||||
"blind_validation": validation,
|
||||
}
|
||||
(output / "match-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(output / "report.html").write_text(html_report(result), encoding="utf-8")
|
||||
print(json.dumps(result["summary"], ensure_ascii=False, indent=2))
|
||||
print(json.dumps({k: validation[k] for k in ("comparable_group_count", "same_name_count", "same_name_rate")}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user