545 lines
21 KiB
Python
545 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import time
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from mapbox_vector_tile.Mapbox import vector_tile_pb2
|
|
from PIL import Image
|
|
|
|
|
|
def canonical(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def stable_id(prefix: str, value: Any) -> str:
|
|
return prefix + "-" + hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
|
|
def protobuf_value(value: Any) -> Any:
|
|
fields = value.ListFields()
|
|
if not fields:
|
|
return None
|
|
return fields[0][1]
|
|
|
|
|
|
def decode_feature_properties(layer: Any, feature: Any) -> dict[str, Any]:
|
|
props: dict[str, Any] = {}
|
|
tags = feature.tags
|
|
for i in range(0, len(tags), 2):
|
|
if i + 1 >= len(tags):
|
|
break
|
|
key_index = tags[i]
|
|
value_index = tags[i + 1]
|
|
if key_index >= len(layer.keys) or value_index >= len(layer.values):
|
|
continue
|
|
props[layer.keys[key_index]] = protobuf_value(layer.values[value_index])
|
|
return props
|
|
|
|
|
|
def geometry_role(feature_type: int) -> str:
|
|
return {1: "Point", 2: "LineString", 3: "Polygon"}.get(int(feature_type), "Unknown")
|
|
|
|
def numeric(value: Any) -> float | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, (int, float)):
|
|
return float(value)
|
|
if isinstance(value, str):
|
|
try:
|
|
return float(value)
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def evaluate(expr: Any, props: dict[str, Any], zoom: int) -> Any:
|
|
if not isinstance(expr, list):
|
|
return expr
|
|
if not expr:
|
|
return expr
|
|
head = expr[0]
|
|
if not isinstance(head, str):
|
|
return [evaluate(x, props, zoom) for x in expr]
|
|
|
|
if head == "get":
|
|
return props.get(expr[1]) if len(expr) > 1 else None
|
|
if head == "has":
|
|
if len(expr) < 2:
|
|
return False
|
|
key = expr[1]
|
|
return key in props and props.get(key) not in (None, "")
|
|
if head == "!":
|
|
return not bool(evaluate(expr[1], props, zoom)) if len(expr) > 1 else True
|
|
if head == "all":
|
|
return all(bool(evaluate(x, props, zoom)) for x in expr[1:])
|
|
if head == "any":
|
|
return any(bool(evaluate(x, props, zoom)) for x in expr[1:])
|
|
if head in {"==", "!=", ">", "<", ">=", "<="}:
|
|
if len(expr) < 3:
|
|
return False
|
|
a = evaluate(expr[1], props, zoom)
|
|
b = evaluate(expr[2], props, zoom)
|
|
if head == "==":
|
|
return a == b
|
|
if head == "!=":
|
|
return a != b
|
|
an = numeric(a)
|
|
bn = numeric(b)
|
|
if an is not None and bn is not None:
|
|
a, b = an, bn
|
|
try:
|
|
return {
|
|
">": a > b,
|
|
"<": a < b,
|
|
">=": a >= b,
|
|
"<=": a <= b,
|
|
}[head]
|
|
except TypeError:
|
|
return False
|
|
if head == "literal":
|
|
return expr[1] if len(expr) > 1 else None
|
|
if head == "to-number":
|
|
value = evaluate(expr[1], props, zoom) if len(expr) > 1 else None
|
|
n = numeric(value)
|
|
if n is not None:
|
|
return n
|
|
return evaluate(expr[2], props, zoom) if len(expr) > 2 else None
|
|
if head == "to-string":
|
|
value = evaluate(expr[1], props, zoom) if len(expr) > 1 else None
|
|
return "" if value is None else str(value)
|
|
if head == "concat":
|
|
parts = []
|
|
for item in expr[1:]:
|
|
value = evaluate(item, props, zoom)
|
|
parts.append("" if value is None else str(value))
|
|
return "".join(parts)
|
|
if head == "coalesce":
|
|
for item in expr[1:]:
|
|
value = evaluate(item, props, zoom)
|
|
if value not in (None, ""):
|
|
return value
|
|
return None
|
|
if head == "case":
|
|
clauses = expr[1:]
|
|
for i in range(0, len(clauses) - 1, 2):
|
|
if bool(evaluate(clauses[i], props, zoom)):
|
|
return evaluate(clauses[i + 1], props, zoom)
|
|
return evaluate(clauses[-1], props, zoom) if clauses else None
|
|
if head == "match":
|
|
if len(expr) < 4:
|
|
return None
|
|
value = evaluate(expr[1], props, zoom)
|
|
arms = expr[2:]
|
|
fallback = arms[-1]
|
|
pairs = arms[:-1]
|
|
for i in range(0, len(pairs), 2):
|
|
if i + 1 >= len(pairs):
|
|
break
|
|
labels, result = pairs[i], pairs[i + 1]
|
|
if isinstance(labels, list):
|
|
matched = value in labels
|
|
else:
|
|
matched = value == labels
|
|
if matched:
|
|
return evaluate(result, props, zoom)
|
|
return evaluate(fallback, props, zoom)
|
|
if head == "zoom":
|
|
return zoom
|
|
if head == "step":
|
|
if len(expr) < 3:
|
|
return None
|
|
input_value = numeric(evaluate(expr[1], props, zoom))
|
|
result = evaluate(expr[2], props, zoom)
|
|
if input_value is None:
|
|
return result
|
|
stops = expr[3:]
|
|
for i in range(0, len(stops), 2):
|
|
if i + 1 >= len(stops):
|
|
break
|
|
stop = numeric(evaluate(stops[i], props, zoom))
|
|
if stop is None:
|
|
continue
|
|
if input_value >= stop:
|
|
result = evaluate(stops[i + 1], props, zoom)
|
|
else:
|
|
break
|
|
return result
|
|
|
|
# Unknown operators stay explicit instead of silently becoming truthy.
|
|
return {"unsupported_expression": expr}
|
|
|
|
|
|
def visible(layer: dict[str, Any], zoom: int) -> bool:
|
|
if zoom < float(layer.get("minzoom", 0)):
|
|
return False
|
|
maxzoom = layer.get("maxzoom")
|
|
if maxzoom is not None and zoom >= float(maxzoom):
|
|
return False
|
|
if (layer.get("layout") or {}).get("visibility") == "none":
|
|
return False
|
|
return True
|
|
|
|
|
|
def matches(layer: dict[str, Any], props: dict[str, Any], zoom: int) -> bool:
|
|
if not visible(layer, zoom):
|
|
return False
|
|
filt = layer.get("filter")
|
|
if filt is None:
|
|
return True
|
|
value = evaluate(filt, props, zoom)
|
|
return value is True
|
|
|
|
|
|
def clean_value(value: Any) -> Any:
|
|
if isinstance(value, float) and math.isfinite(value):
|
|
return round(value, 6)
|
|
return value
|
|
|
|
|
|
def sprite_hashes(meta: dict[str, Any], atlas: Image.Image) -> dict[str, str]:
|
|
out = {}
|
|
for key, m in meta.items():
|
|
crop = atlas.crop((
|
|
int(m["x"]), int(m["y"]),
|
|
int(m["x"]) + int(m["width"]),
|
|
int(m["y"]) + int(m["height"]),
|
|
))
|
|
out[key] = hashlib.sha256(crop.tobytes()).hexdigest()
|
|
return out
|
|
|
|
|
|
def component_from_layer(
|
|
layer: dict[str, Any],
|
|
style_index: int,
|
|
props: dict[str, Any],
|
|
zoom: int,
|
|
sprite_meta: dict[str, Any],
|
|
visual_hashes: dict[str, str],
|
|
) -> list[dict[str, Any]]:
|
|
ltype = layer.get("type")
|
|
layout = layer.get("layout") or {}
|
|
paint = layer.get("paint") or {}
|
|
out: list[dict[str, Any]] = []
|
|
|
|
if ltype == "symbol":
|
|
if "icon-image" in layout:
|
|
key = evaluate(layout["icon-image"], props, zoom)
|
|
if isinstance(key, str) and key:
|
|
out.append({
|
|
"style_index": style_index,
|
|
"style_layer_id": layer.get("id"),
|
|
"role": "icon",
|
|
"resource_key": key,
|
|
"resource_visual_hash": visual_hashes.get(key),
|
|
"resource_present": key in sprite_meta,
|
|
"icon_size": clean_value(evaluate(layout.get("icon-size", 1.0), props, zoom)),
|
|
"icon_rotate": clean_value(evaluate(layout.get("icon-rotate", 0.0), props, zoom)),
|
|
"icon_anchor": clean_value(evaluate(layout.get("icon-anchor", "center"), props, zoom)),
|
|
})
|
|
if "text-field" in layout:
|
|
text_value = evaluate(layout["text-field"], props, zoom)
|
|
if text_value not in (None, ""):
|
|
out.append({
|
|
"style_index": style_index,
|
|
"style_layer_id": layer.get("id"),
|
|
"role": "text",
|
|
"text_present": True,
|
|
"text_size": clean_value(evaluate(layout.get("text-size"), props, zoom)),
|
|
"text_anchor": clean_value(evaluate(layout.get("text-anchor"), props, zoom)),
|
|
"text_offset": clean_value(evaluate(layout.get("text-offset"), props, zoom)),
|
|
"text_color": clean_value(evaluate(paint.get("text-color"), props, zoom)),
|
|
"text_halo_color": clean_value(evaluate(paint.get("text-halo-color"), props, zoom)),
|
|
"text_halo_width": clean_value(evaluate(paint.get("text-halo-width"), props, zoom)),
|
|
})
|
|
|
|
elif ltype == "line":
|
|
key = evaluate(paint.get("line-pattern"), props, zoom) if "line-pattern" in paint else None
|
|
out.append({
|
|
"style_index": style_index,
|
|
"style_layer_id": layer.get("id"),
|
|
"role": "line",
|
|
"resource_key": key if isinstance(key, str) and key else None,
|
|
"resource_visual_hash": visual_hashes.get(key) if isinstance(key, str) else None,
|
|
"resource_present": key in sprite_meta if isinstance(key, str) else None,
|
|
"line_color": clean_value(evaluate(paint.get("line-color"), props, zoom)),
|
|
"line_width": clean_value(evaluate(paint.get("line-width"), props, zoom)),
|
|
"line_dasharray": clean_value(evaluate(paint.get("line-dasharray"), props, zoom)),
|
|
"line_opacity": clean_value(evaluate(paint.get("line-opacity"), props, zoom)),
|
|
})
|
|
|
|
elif ltype == "fill":
|
|
key = evaluate(paint.get("fill-pattern"), props, zoom) if "fill-pattern" in paint else None
|
|
out.append({
|
|
"style_index": style_index,
|
|
"style_layer_id": layer.get("id"),
|
|
"role": "fill",
|
|
"resource_key": key if isinstance(key, str) and key else None,
|
|
"resource_visual_hash": visual_hashes.get(key) if isinstance(key, str) else None,
|
|
"resource_present": key in sprite_meta if isinstance(key, str) else None,
|
|
"fill_color": clean_value(evaluate(paint.get("fill-color"), props, zoom)),
|
|
"fill_outline_color": clean_value(evaluate(paint.get("fill-outline-color"), props, zoom)),
|
|
"fill_opacity": clean_value(evaluate(paint.get("fill-opacity"), props, zoom)),
|
|
})
|
|
|
|
elif ltype == "circle":
|
|
out.append({
|
|
"style_index": style_index,
|
|
"style_layer_id": layer.get("id"),
|
|
"role": "circle",
|
|
"circle_color": clean_value(evaluate(paint.get("circle-color"), props, zoom)),
|
|
"circle_radius": clean_value(evaluate(paint.get("circle-radius"), props, zoom)),
|
|
"circle_stroke_color": clean_value(evaluate(paint.get("circle-stroke-color"), props, zoom)),
|
|
"circle_stroke_width": clean_value(evaluate(paint.get("circle-stroke-width"), props, zoom)),
|
|
})
|
|
|
|
return out
|
|
|
|
|
|
def signature_components(components: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
# Exclude provider/resource names and Style layer IDs from the visual signature.
|
|
result = []
|
|
for c in components:
|
|
x = {k: v for k, v in c.items() if k not in {"style_layer_id", "resource_key"}}
|
|
result.append(x)
|
|
return result
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--style", required=True)
|
|
ap.add_argument("--sprite-json", required=True)
|
|
ap.add_argument("--sprite-png", required=True)
|
|
ap.add_argument("--pbf-root", required=True)
|
|
ap.add_argument("--output", required=True)
|
|
ap.add_argument("--provider-id", required=True)
|
|
ap.add_argument("--dataset-id", required=True)
|
|
ap.add_argument("--provider-version", default="unknown")
|
|
ap.add_argument("--catalog-name")
|
|
ap.add_argument("--progress-every", type=int, default=5000)
|
|
args = ap.parse_args()
|
|
|
|
style = json.loads(Path(args.style).read_text(encoding="utf-8"))
|
|
sprite_meta = json.loads(Path(args.sprite_json).read_text(encoding="utf-8"))
|
|
atlas = Image.open(args.sprite_png).convert("RGBA")
|
|
visual_hashes = sprite_hashes(sprite_meta, atlas)
|
|
|
|
indexed: dict[str, list[tuple[int, dict[str, Any]]]] = defaultdict(list)
|
|
for i, layer in enumerate(style.get("layers", [])):
|
|
sl = layer.get("source-layer")
|
|
if sl:
|
|
indexed[sl].append((i, layer))
|
|
|
|
root = Path(args.pbf_root)
|
|
paths = sorted(root.glob("*/*/*.pbf"))
|
|
output = Path(args.output)
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
catalog_name = args.catalog_name or f"{args.provider_id}-actual-anonymous"
|
|
|
|
distinct: dict[str, dict[str, Any]] = {}
|
|
group_stats: dict[str, Counter] = defaultdict(Counter)
|
|
source_group_map: dict[str, str] = {}
|
|
layer_seen = Counter()
|
|
feature_count = 0
|
|
matched_feature_count = 0
|
|
unsupported_count = 0
|
|
missing_sprite_count = 0
|
|
occurrences: list[dict[str, Any]] = []
|
|
started = time.time()
|
|
|
|
for tile_idx, path in enumerate(paths, 1):
|
|
parts = path.relative_to(root).parts
|
|
z, x, yfile = int(parts[0]), int(parts[1]), parts[2]
|
|
y = int(Path(yfile).stem)
|
|
try:
|
|
decoded = vector_tile_pb2.tile()
|
|
decoded.ParseFromString(path.read_bytes())
|
|
except Exception as exc:
|
|
print(f"DECODE_FAIL {path}: {exc}", flush=True)
|
|
continue
|
|
|
|
for tile_layer in decoded.layers:
|
|
layer_name = tile_layer.name
|
|
layer_seen[layer_name] += len(tile_layer.features)
|
|
style_layers = indexed.get(layer_name, [])
|
|
if not style_layers:
|
|
continue
|
|
group_id = source_group_map.setdefault(
|
|
layer_name,
|
|
stable_id("GRP", {"provider_id": args.provider_id, "source_layer": layer_name}),
|
|
)
|
|
for feature in tile_layer.features:
|
|
feature_count += 1
|
|
props = decode_feature_properties(tile_layer, feature)
|
|
components: list[dict[str, Any]] = []
|
|
for style_index, style_layer in style_layers:
|
|
if not matches(style_layer, props, z):
|
|
continue
|
|
comps = component_from_layer(
|
|
style_layer, style_index, props, z, sprite_meta, visual_hashes
|
|
)
|
|
for comp in comps:
|
|
if isinstance(comp, dict):
|
|
if any(
|
|
isinstance(v, dict) and "unsupported_expression" in v
|
|
for v in comp.values()
|
|
):
|
|
unsupported_count += 1
|
|
if comp.get("resource_key") and comp.get("resource_present") is False:
|
|
missing_sprite_count += 1
|
|
components.extend(comps)
|
|
|
|
if not components:
|
|
continue
|
|
matched_feature_count += 1
|
|
visual_signature = {
|
|
"geometry_role": geometry_role(feature.type),
|
|
"components": signature_components(components),
|
|
}
|
|
vo_id = stable_id("VO", visual_signature)
|
|
feature_id = int(feature.id) if feature.HasField("id") else None
|
|
tile_id = f"{z}/{x}/{y}"
|
|
occurrence_id = stable_id("OCC", {
|
|
"tile": tile_id,
|
|
"source_layer": layer_name,
|
|
"feature_id": feature_id,
|
|
"feature_ordinal": feature_count,
|
|
"visual_object_id": vo_id,
|
|
})
|
|
item = distinct.get(vo_id)
|
|
if item is None:
|
|
item = {
|
|
"visual_object_id": vo_id,
|
|
"group_id": group_id,
|
|
"geometry_role": visual_signature["geometry_role"],
|
|
"components": components,
|
|
"occurrence_count": 0,
|
|
"provenance": {
|
|
"source_layer": layer_name,
|
|
},
|
|
"example": {
|
|
"tile": tile_id,
|
|
"feature_id": feature_id,
|
|
},
|
|
}
|
|
distinct[vo_id] = item
|
|
item["occurrence_count"] += 1
|
|
occurrences.append({
|
|
"occurrence_id": occurrence_id,
|
|
"visual_object_id": vo_id,
|
|
"tile": tile_id,
|
|
"feature_id": feature_id,
|
|
"feature_ordinal": feature_count,
|
|
"source": style_layers[0][1].get("source") if style_layers else None,
|
|
"source_layer": layer_name,
|
|
"matched_style_layers": [c.get("style_layer_id") for c in components],
|
|
"components": signature_components(components),
|
|
"geometry_role": geometry_role(feature.type),
|
|
"zoom": z,
|
|
"field_evidence": decode_feature_properties(tile_layer, feature),
|
|
})
|
|
group_stats[group_id]["tile_feature_occurrences"] += 1
|
|
group_stats[group_id]["distinct_vo_ids"] = 0
|
|
|
|
if tile_idx % args.progress_every == 0 or tile_idx == len(paths):
|
|
elapsed = time.time() - started
|
|
print(
|
|
f"PROGRESS tiles={tile_idx}/{len(paths)} features={feature_count} "
|
|
f"distinct_vo={len(distinct)} elapsed={elapsed:.1f}s",
|
|
flush=True,
|
|
)
|
|
|
|
# Count distinct VOs per group after the scan.
|
|
per_group_distinct = Counter(item["group_id"] for item in distinct.values())
|
|
groups = []
|
|
for source_layer, group_id in sorted(source_group_map.items()):
|
|
groups.append({
|
|
"group_id": group_id,
|
|
"distinct_visual_object_count": per_group_distinct[group_id],
|
|
"tile_feature_occurrences": group_stats[group_id]["tile_feature_occurrences"],
|
|
"provenance": {"source_layer": source_layer},
|
|
})
|
|
|
|
catalog = {
|
|
"schema": "actual-visual-object-catalog/v1",
|
|
"catalog_name": catalog_name,
|
|
"principle": (
|
|
"PBF is used only to instantiate Style visual expressions and occurrence combinations; "
|
|
"provider semantic names are not used for classification."
|
|
),
|
|
"inputs": {
|
|
"provider_id": args.provider_id,
|
|
"dataset_id": args.dataset_id,
|
|
"provider_version": args.provider_version,
|
|
"style": str(Path(args.style)),
|
|
"sprite_json": str(Path(args.sprite_json)),
|
|
"sprite_png": str(Path(args.sprite_png)),
|
|
"pbf_root": str(root),
|
|
"tile_count": len(paths),
|
|
},
|
|
"summary": {
|
|
"decoded_feature_count_in_styled_layers": feature_count,
|
|
"matched_feature_occurrence_count": matched_feature_count,
|
|
"distinct_visual_object_count": len(distinct),
|
|
"group_count": len(groups),
|
|
"unsupported_component_value_count": unsupported_count,
|
|
"missing_sprite_component_count": missing_sprite_count,
|
|
"source_layer_occurrence_counts": dict(layer_seen),
|
|
},
|
|
"groups": groups,
|
|
"visual_objects": sorted(distinct.values(), key=lambda x: x["visual_object_id"]),
|
|
"occurrences": occurrences,
|
|
}
|
|
(output / "actual-anonymous-catalog.json").write_text(
|
|
json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
(output / "visual-object-occurrences.json").write_text(
|
|
json.dumps({
|
|
"schema": "visual-object-occurrences/v1",
|
|
"provider_id": args.provider_id,
|
|
"dataset_id": args.dataset_id,
|
|
"occurrences": occurrences,
|
|
}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
(output / "instantiation-report.json").write_text(
|
|
json.dumps({
|
|
"schema": "visual-object-instantiation-report/v1",
|
|
"provider_id": args.provider_id,
|
|
"dataset_id": args.dataset_id,
|
|
"provider_version": args.provider_version,
|
|
"inputs": catalog["inputs"],
|
|
"summary": catalog["summary"],
|
|
"occurrence_count": len(occurrences),
|
|
}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
sanitized = {
|
|
"schema": catalog["schema"],
|
|
"catalog_name": catalog["catalog_name"],
|
|
"occurrences": catalog["occurrences"],
|
|
"summary": catalog["summary"],
|
|
"groups": [
|
|
{k: v for k, v in g.items() if k != "provenance"}
|
|
for g in catalog["groups"]
|
|
],
|
|
"visual_objects": [
|
|
{k: v for k, v in obj.items() if k not in {"provenance", "example"}}
|
|
for obj in catalog["visual_objects"]
|
|
],
|
|
}
|
|
(output / "matcher-input-sanitized.json").write_text(
|
|
json.dumps(sanitized, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
print(json.dumps(catalog["summary"], ensure_ascii=False, indent=2), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|