251 lines
13 KiB
Python
251 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""PBF 原始 Feature 身份作用域审计（只读；绝不授予 stable/global identity）。"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
from collections import Counter, defaultdict
|
||
import hashlib
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from mapbox_vector_tile.Mapbox import vector_tile_pb2
|
||
|
||
|
||
def _tile_parts(value: Any) -> tuple[int, int, int] | None:
|
||
if not isinstance(value, str):
|
||
return None
|
||
parts = value.split("/")
|
||
if len(parts) != 3 or any(not x.isdigit() for x in parts):
|
||
return None
|
||
z, x, y = map(int, parts)
|
||
if z > 24 or x >= (1 << z) or y >= (1 << z):
|
||
return None
|
||
return z, x, y
|
||
|
||
|
||
def _property_signature(tile_layer: Any, feature: Any) -> str:
|
||
"""Canonicalize original protobuf properties without Provider field semantics."""
|
||
tags = feature.tags
|
||
props: dict[str, Any] = {}
|
||
for i in range(0, len(tags) - 1, 2):
|
||
key_i, value_i = tags[i], tags[i + 1]
|
||
if key_i >= len(tile_layer.keys) or value_i >= len(tile_layer.values):
|
||
continue
|
||
fields = tile_layer.values[value_i].ListFields()
|
||
props[tile_layer.keys[key_i]] = fields[0][1] if fields else None
|
||
serialized = json.dumps(props, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
||
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _source_owners(style: dict[str, Any]) -> dict[str, set[str]]:
|
||
result: dict[str, set[str]] = defaultdict(set)
|
||
for layer in style.get("layers", []):
|
||
if isinstance(layer, dict) and isinstance(layer.get("source-layer"), str):
|
||
result[layer["source-layer"]].add(layer.get("source"))
|
||
return result
|
||
|
||
|
||
def audit_scope(
|
||
catalog: dict[str, Any], style: dict[str, Any], pbf_root: Path,
|
||
*, provider_id: str, dataset_id: str,
|
||
mapping: dict[str, Any] | None = None,
|
||
sample_occurrences: list[dict[str, Any]] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""One-example-per-type locator proof; explicit refusal of stable identity."""
|
||
objects = catalog.get("visual_objects")
|
||
if not isinstance(objects, list) or not objects:
|
||
raise ValueError("Catalog 缺少 visual_objects")
|
||
owners = _source_owners(style)
|
||
by_tile: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
seen_types: set[str] = set()
|
||
for obj in objects:
|
||
object_id = obj.get("visual_object_id")
|
||
if not isinstance(object_id, str) or not object_id or object_id in seen_types:
|
||
raise ValueError("Visual Object 类型 ID 重复或无效")
|
||
seen_types.add(object_id)
|
||
example = obj.get("example")
|
||
sl = (obj.get("provenance") or {}).get("source_layer")
|
||
if not isinstance(example, dict) or _tile_parts(example.get("tile")) is None:
|
||
raise ValueError(f"{object_id}: 样本 Tile 路径不可信")
|
||
if not isinstance(sl, str) or not sl:
|
||
raise ValueError(f"{object_id}: Source Layer 缺失")
|
||
by_tile[example["tile"]].append(obj)
|
||
if mapping is not None:
|
||
assignments = mapping.get("assignments")
|
||
if not isinstance(assignments, list):
|
||
raise ValueError("Mapping assignments 无效")
|
||
assignment_by_id = {row.get("visualObjectId"): row for row in assignments}
|
||
if len(assignment_by_id) != len(assignments):
|
||
raise ValueError("Mapping Visual Object ID 重复")
|
||
else:
|
||
assignment_by_id = {}
|
||
results: list[dict[str, Any]] = []
|
||
sample_stats = Counter()
|
||
raw_tile_features = Counter()
|
||
cross_tile_ids: dict[tuple[str, str, int], set[str]] = defaultdict(set)
|
||
raw_property_signatures: dict[tuple[str, str, int], set[str]] = defaultdict(set)
|
||
observed_local_keys: dict[tuple[str, str, int], int] = {}
|
||
for tile in sorted(by_tile, key=lambda t: _tile_parts(t)):
|
||
z, x, y = _tile_parts(tile)
|
||
file = pbf_root / str(z) / str(x) / (str(y) + ".pbf")
|
||
lookup: dict[str, Any] = {}
|
||
same_name: set[str] = set()
|
||
if file.is_file():
|
||
decoded = vector_tile_pb2.tile()
|
||
decoded.ParseFromString(file.read_bytes())
|
||
layer_counts = Counter(layer.name for layer in decoded.layers)
|
||
same_name = {name for name, count in layer_counts.items() if count > 1}
|
||
lookup = {layer.name: layer for layer in decoded.layers}
|
||
for layer in decoded.layers:
|
||
local_ids = Counter(int(f.id) for f in layer.features if f.HasField("id"))
|
||
raw_tile_features["decodedFeatures"] += len(layer.features)
|
||
raw_tile_features["missingFeatureIds"] += sum(not f.HasField("id") for f in layer.features)
|
||
raw_tile_features["duplicateFeatureIdValues"] += sum(v > 1 for v in local_ids.values())
|
||
for f in layer.features:
|
||
if f.HasField("id"):
|
||
identity = (layer.name, "rawFeatureId", int(f.id))
|
||
cross_tile_ids[identity].add(tile)
|
||
raw_property_signatures[identity].add(_property_signature(layer, f))
|
||
for obj in by_tile[tile]:
|
||
sl = obj["provenance"]["source_layer"]
|
||
example = obj["example"]
|
||
fid = example.get("feature_id")
|
||
sources = owners.get(sl, set())
|
||
status = "TILE_LOCAL_LOCATOR_CONFIRMED"
|
||
source_id = next(iter(sources)) if len(sources) == 1 else None
|
||
if source_id is None or not isinstance(source_id, str):
|
||
status = "STYLE_SOURCE_SCOPE_UNPROVEN"
|
||
elif not file.is_file():
|
||
status = "PBF_TILE_MISSING"
|
||
elif sl in same_name:
|
||
status = "DUPLICATE_SOURCE_LAYER_IN_TILE"
|
||
elif sl not in lookup:
|
||
status = "SOURCE_LAYER_IN_TILE_MISSING"
|
||
elif type(fid) is not int or fid < 0:
|
||
status = "FEATURE_ID_UNAVAILABLE"
|
||
else:
|
||
count = sum(f.HasField("id") and int(f.id) == fid for f in lookup[sl].features)
|
||
if count != 1:
|
||
status = "FEATURE_ID_MISSING" if not count else "FEATURE_ID_AMBIGUOUS"
|
||
if status == "TILE_LOCAL_LOCATOR_CONFIRMED":
|
||
key = (tile, sl, fid)
|
||
if key in observed_local_keys:
|
||
# Invalidate both the original and subsequent claim; never
|
||
# leave the first conflicting Visual Object "confirmed".
|
||
status = "EXAMPLE_LOCATOR_SHARED_BY_VISUAL_TYPES"
|
||
previous = results[observed_local_keys[key]]
|
||
if previous["locatorStatus"] == "TILE_LOCAL_LOCATOR_CONFIRMED":
|
||
previous["locatorStatus"] = status
|
||
previous["identityScope"] = "UNVERIFIED"
|
||
previous["tileLocalFeatureLocator"] = None
|
||
sample_stats["TILE_LOCAL_LOCATOR_CONFIRMED"] -= 1
|
||
sample_stats[status] += 1
|
||
else:
|
||
observed_local_keys[key] = len(results)
|
||
sample_stats[status] += 1
|
||
assignment = assignment_by_id.get(obj["visual_object_id"])
|
||
components = obj.get("components", [])
|
||
results.append({
|
||
"visualObjectTypeId": obj["visual_object_id"],
|
||
"identityScope": "TILE_LOCAL_EXAMPLE_ONLY" if status == "TILE_LOCAL_LOCATOR_CONFIRMED" else "UNVERIFIED",
|
||
"locatorStatus": status,
|
||
"tileLocalFeatureLocator": {
|
||
"providerId": provider_id, "datasetId": dataset_id,
|
||
"sourceId": source_id, "sourceLayer": sl,
|
||
"z": z, "x": x, "y": y, "rawFeatureId": fid,
|
||
} if status == "TILE_LOCAL_LOCATOR_CONFIRMED" else None,
|
||
"stableEntityId": None,
|
||
"fidScopePromotionAuthorized": False,
|
||
"pickupDeduplicationAuthorized": False,
|
||
"assignmentStatus": assignment.get("status") if isinstance(assignment, dict) else None,
|
||
"standardLayer": assignment.get("standardLayer") if isinstance(assignment, dict) else None,
|
||
"primaryReviewStatus": "PENDING_REVIEW",
|
||
"primaryStyleComponentIndex": None,
|
||
"componentCandidates": [
|
||
{"componentIndex": i, "styleLayerId": c.get("style_layer_id"),
|
||
"styleIndex": c.get("style_index"), "drawingRole": c.get("role"),
|
||
"resourceKey": c.get("resource_key")}
|
||
for i, c in enumerate(components)
|
||
],
|
||
})
|
||
sampled_ids: dict[tuple[str, str, int], set[str]] = defaultdict(set)
|
||
if sample_occurrences is not None:
|
||
for occ in sample_occurrences:
|
||
fid = occ.get("feature_id")
|
||
if type(fid) is int and isinstance(occ.get("source_layer"), str) and _tile_parts(occ.get("tile")):
|
||
sampled_ids[(occ["source_layer"], "rawFeatureId", fid)].add(occ["tile"])
|
||
return {
|
||
"schemaVersion": "navsea-feature-identity-scope-audit/v1",
|
||
"scope": "RECORDED_EXAMPLE_PER_VISUAL_TYPE_NOT_GLOBAL_ENTITY_VERIFICATION",
|
||
"providerId": provider_id,
|
||
"datasetId": dataset_id,
|
||
"visualObjectCount": len(results),
|
||
"examinedUniqueTileCount": len(by_tile),
|
||
"exampleLocatorStatusCounts": dict(sorted((key, value) for key, value in sample_stats.items() if value > 0)),
|
||
"examinedTileRawFeatureStats": dict(sorted(raw_tile_features.items())),
|
||
"reusedRawFeatureIdAcrossExaminedTiles": sum(len(v) > 1 for v in cross_tile_ids.values()),
|
||
"reusedRawFeatureIdWithDifferingProperties": sum(
|
||
len(tiles) > 1 and len(raw_property_signatures[identity]) > 1
|
||
for identity, tiles in cross_tile_ids.items()
|
||
),
|
||
"sampleOccurrencesWithSameIdAcrossTiles": sum(len(v) > 1 for v in sampled_ids.values()),
|
||
"fidScope": "unknown",
|
||
"globalIdentityAuthorizedCount": 0,
|
||
"primaryAuthorizedCount": 0,
|
||
"pickupDeduplicationAuthorizedCount": 0,
|
||
"reviewQueue": sorted(results, key=lambda x: x["visualObjectTypeId"]),
|
||
}
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--catalog", type=Path, required=True)
|
||
parser.add_argument("--style", type=Path, required=True)
|
||
parser.add_argument("--pbf-root", type=Path, required=True)
|
||
parser.add_argument("--mapping", type=Path)
|
||
parser.add_argument("--sample-occurrences", type=Path)
|
||
parser.add_argument("--provider-id", required=True)
|
||
parser.add_argument("--dataset-id", required=True)
|
||
parser.add_argument("--output", type=Path, required=True)
|
||
args = parser.parse_args()
|
||
catalog = json.loads(args.catalog.read_text(encoding="utf-8"))
|
||
if catalog.get("schema") != "actual-visual-object-catalog/v1":
|
||
raise ValueError("Catalog schema 不匹配")
|
||
style_data = args.style.read_bytes()
|
||
original_style = (catalog.get("inputs") or {}).get("style")
|
||
if not isinstance(original_style, str) or not Path(original_style).is_file():
|
||
raise ValueError("Catalog 原始 Style 缺失，无法核验")
|
||
if hashlib.sha256(style_data).digest() != hashlib.sha256(Path(original_style).read_bytes()).digest():
|
||
raise ValueError("Catalog 与待查 Style SHA-256 不一致")
|
||
mapping = json.loads(args.mapping.read_text(encoding="utf-8")) if args.mapping else None
|
||
occurrences = None
|
||
if args.sample_occurrences:
|
||
raw = json.loads(args.sample_occurrences.read_text(encoding="utf-8"))
|
||
occurrences = raw.get("occurrences")
|
||
if not isinstance(occurrences, list):
|
||
raise ValueError("样本 occurrences 数据无效")
|
||
result = audit_scope(
|
||
catalog, json.loads(style_data), args.pbf_root,
|
||
provider_id=args.provider_id, dataset_id=args.dataset_id,
|
||
mapping=mapping, sample_occurrences=occurrences,
|
||
)
|
||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(json.dumps({
|
||
"visualObjectCount": result["visualObjectCount"],
|
||
"examinedUniqueTileCount": result["examinedUniqueTileCount"],
|
||
"exampleLocatorStatusCounts": result["exampleLocatorStatusCounts"],
|
||
"examinedTileRawFeatureStats": result["examinedTileRawFeatureStats"],
|
||
"reusedRawFeatureIdAcrossExaminedTiles": result["reusedRawFeatureIdAcrossExaminedTiles"],
|
||
"reusedRawFeatureIdWithDifferingProperties": result["reusedRawFeatureIdWithDifferingProperties"],
|
||
"fidScope": result["fidScope"],
|
||
"globalIdentityAuthorizedCount": 0,
|
||
"output": str(args.output),
|
||
}, ensure_ascii=False))
|
||
return 0 if result["exampleLocatorStatusCounts"] == {"TILE_LOCAL_LOCATOR_CONFIRMED": result["visualObjectCount"]} else 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|