tei/pbf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: 全量快照提交以防磁盘风险

Browse Source
This commit is contained in:
OpenAI Codex
2026-04-08 19:32:25 +08:00
parent 86726e3247
commit 4f96d7be91
243 changed files with 49757 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

509
navsea_render_audit.py
View File

@@ -31,6 +31,8 @@ from typing import Any
import mapbox_vector_tile
import pymysql
from navsea_tile_builder import NavSeaFidCodec
DEFAULT_ORIGINAL_STYLE_PATH = Path("/mnt/sda1/www/newpec/style.patched.local.json")
DEFAULT_ENGINEERING_STYLE_PATH = Path("/mnt/sda1/www/newpec/navsea-engineering.json")
@@ -52,6 +54,10 @@ ENGINEERING_TILE_ROOT = DEFAULT_ENGINEERING_TILE_ROOT
REPORT_MD_PATH = DEFAULT_REPORT_MD_PATH
REPORT_JSON_PATH = DEFAULT_REPORT_JSON_PATH
AUDIT_NAME = DEFAULT_AUDIT_NAME
SKIP_DB_PERSIST = False
COMPARISON_LABEL = "工程版"
FID_CODEC: NavSeaFidCodec | None = None
MATCH_ON_FID_ONLY = False
DERIVED_PROPERTY_PREFIXES = (
"canonical_",
@@ -120,6 +126,27 @@ EXPRESSION_OPS = {
"zoom",
}
DEPTH_NUMERIC_SOURCE_LAYERS = {
"L海底地形",
"L概略等深線",
"L等深線",
}
CLEARANCE_NUMERIC_SOURCE_LAYERS = {
"p高さ制限",
}
SAFETY_ICON_SOURCE_LAYERS = {
"p航路標識群",
"p航路標識",
"P航路標識",
"p灯台",
"p灯浮標",
"p灯立標",
"p灯標",
}
SUPPORT_FILL_SOURCE_LAYERS = {
"P穴",
}
@dataclass(frozen=True)
class DbConfig:
@@ -141,6 +168,7 @@ class AuditFeature:
geom_type: str
object_id: str
fid_legacy: str | None
native_feature_id: int | None
properties: dict[str, Any]
@@ -495,21 +523,59 @@ def geometry_hash(geometry: dict[str, Any]) -> str:
return hash_text(canonical_json(geometry))
def extract_legacy_fid(properties: dict[str, Any]) -> str | None:
def extract_legacy_fid(properties: dict[str, Any], native_feature_id: int | None = None) -> str | None:
legacy = properties.get("fid_legacy_raw")
if legacy not in (None, ""):
return str(legacy)
try:
return str(int(str(legacy)))
except ValueError:
return str(legacy)
fid = properties.get("fid")
if fid not in (None, ""):
return str(fid)
fid_text = str(fid).strip()
# Original tiles store legacy fid as decimal integers. Only treat
# an 8-char property fid as encrypted hex when it is not plain digits.
if fid_text.isdigit():
return str(int(fid_text))
if FID_CODEC is not None and len(fid_text) == 8:
try:
decoded = FID_CODEC.decrypt_number(int(fid_text, 16))
return str(decoded)
except ValueError:
pass
try:
return str(int(fid_text))
except ValueError:
return fid_text
if native_feature_id is not None:
if FID_CODEC is not None:
try:
decoded = FID_CODEC.decrypt_number(int(native_feature_id))
return str(decoded)
except ValueError:
pass
try:
feature_id_int = int(native_feature_id)
if feature_id_int > 2147483647:
feature_id_int -= 4294967296
return str(feature_id_int)
except ValueError:
return str(native_feature_id)
return None
def object_identity(layer_name: str, geometry: dict[str, Any], properties: dict[str, Any]) -> tuple[str, str | None]:
def object_identity(
layer_name: str,
geometry: dict[str, Any],
properties: dict[str, Any],
native_feature_id: int | None = None,
) -> tuple[str, str | None]:
geom_type = str(geometry.get("type", ""))
legacy_source_layer = text_or_none(properties.get("source_layer_jp")) or layer_name
legacy_fid = extract_legacy_fid(properties)
legacy_fid = extract_legacy_fid(properties, native_feature_id)
if legacy_fid:
if MATCH_ON_FID_ONLY:
return f"fid:{legacy_fid}|geom:{geom_type}", legacy_fid
return f"fid:{legacy_fid}|src:{legacy_source_layer}|geom:{geom_type}", legacy_fid
fallback_payload = {
"source_layer": legacy_source_layer,
@@ -521,6 +587,8 @@ def object_identity(layer_name: str, geometry: dict[str, Any], properties: dict[
def feature_instance_id(feature: AuditFeature) -> str:
if MATCH_ON_FID_ONLY:
return f"{feature.object_id}|z:{feature.tile_z}|x:{feature.tile_x}|y:{feature.tile_y}"
return (
f"{feature.object_id}|z:{feature.tile_z}|x:{feature.tile_x}|"
f"y:{feature.tile_y}|layer:{feature.layer_name}"
@@ -538,7 +606,9 @@ def decode_tile_instances(dataset: str, tile_root: Path, style: dict[str, Any],
for feature in payload.get("features", []):
geometry = feature.get("geometry") or {}
properties = dict(feature.get("properties") or {})
object_id, legacy_fid = object_identity(layer_name, geometry, properties)
native_feature_id = feature.get("id")
native_feature_id = int(native_feature_id) if isinstance(native_feature_id, int) else None
object_id, legacy_fid = object_identity(layer_name, geometry, properties, native_feature_id)
audit_feature = AuditFeature(
dataset=dataset,
tile_z=z,
@@ -548,6 +618,7 @@ def decode_tile_instances(dataset: str, tile_root: Path, style: dict[str, Any],
geom_type=str(geometry.get("type", "")),
object_id=object_id,
fid_legacy=legacy_fid,
native_feature_id=native_feature_id,
properties=properties,
)
instance_id = feature_instance_id(audit_feature)
@@ -565,16 +636,39 @@ def summarize_observations(observations: list[RenderObservation]) -> dict[str, l
return {key: sorted(values) for key, values in grouped.items()}
def summarize_observation_payloads(observations: list[RenderObservation]) -> dict[str, list[dict[str, Any]]]:
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for observation in observations:
grouped[observation.component_type].append(dict(observation.payload))
return {
key: sorted(values, key=canonical_json)
for key, values in grouped.items()
}
def normalize_component_map(component_map: dict[str, list[str]]) -> dict[str, tuple[str, ...]]:
return {key: tuple(values) for key, values in sorted(component_map.items())}
def normalize_payload_map(payload_map: dict[str, list[dict[str, Any]]]) -> dict[str, tuple[dict[str, Any], ...]]:
return {
key: tuple(sorted(values, key=canonical_json))
for key, values in sorted(payload_map.items())
}
def compare_instance(
original: dict[str, Any] | None,
engineering: dict[str, Any] | None,
) -> dict[str, Any]:
if original is None:
eng_feature = engineering["feature"]
engineering_components = normalize_component_map(
summarize_observations(engineering["observations"])
)
engineering_payloads = normalize_payload_map(
summarize_observation_payloads(engineering["observations"])
)
return {
"status": "extra_in_engineering",
"object_instance_id": feature_instance_id(eng_feature),
@@ -587,12 +681,18 @@ def compare_instance(
"source_layer": eng_feature.layer_name,
"canonical_object_type": text_or_none(eng_feature.properties.get("canonical_object_type")),
"original_components": {},
"engineering_components": normalize_component_map(
summarize_observations(engineering["observations"])
),
"engineering_components": engineering_components,
"original_payloads": {},
"engineering_payloads": engineering_payloads,
}
if engineering is None:
orig_feature = original["feature"]
original_components = normalize_component_map(
summarize_observations(original["observations"])
)
original_payloads = normalize_payload_map(
summarize_observation_payloads(original["observations"])
)
return {
"status": "missing_in_engineering",
"object_instance_id": feature_instance_id(orig_feature),
@@ -604,16 +704,18 @@ def compare_instance(
"tile_y": orig_feature.tile_y,
"source_layer": orig_feature.layer_name,
"canonical_object_type": None,
"original_components": normalize_component_map(
summarize_observations(original["observations"])
),
"original_components": original_components,
"engineering_components": {},
"original_payloads": original_payloads,
"engineering_payloads": {},
}
orig_feature = original["feature"]
eng_feature = engineering["feature"]
original_components = normalize_component_map(summarize_observations(original["observations"]))
engineering_components = normalize_component_map(summarize_observations(engineering["observations"]))
original_payloads = normalize_payload_map(summarize_observation_payloads(original["observations"]))
engineering_payloads = normalize_payload_map(summarize_observation_payloads(engineering["observations"]))
if original_components == engineering_components:
status = "exact_match"
@@ -637,6 +739,8 @@ def compare_instance(
"canonical_object_type": text_or_none(eng_feature.properties.get("canonical_object_type")),
"original_components": original_components,
"engineering_components": engineering_components,
"original_payloads": original_payloads,
"engineering_payloads": engineering_payloads,
}
@@ -650,6 +754,101 @@ def format_component_map(component_map: dict[str, tuple[str, ...]] | dict[str, l
return "; ".join(parts)
def has_component(component_map: dict[str, tuple[str, ...]] | dict[str, list[str]], component_type: str) -> bool:
values = component_map.get(component_type)
return bool(values)
def get_component_payloads(
payload_map: dict[str, tuple[dict[str, Any], ...]] | dict[str, list[dict[str, Any]]],
component_type: str,
) -> list[dict[str, Any]]:
values = payload_map.get(component_type) or ()
return [dict(value) for value in values]
def payload_has_nontransparent_fill(payloads: list[dict[str, Any]]) -> bool:
for payload in payloads:
color = text_or_none(payload.get("fill_color"))
opacity = payload.get("fill_opacity")
if opacity == 0:
continue
if color in (None, "rgba(0,0,0,0)", "rgba(0, 0, 0, 0)"):
continue
return True
return False
def payload_has_transparent_fill(payloads: list[dict[str, Any]]) -> bool:
for payload in payloads:
color = text_or_none(payload.get("fill_color"))
opacity = payload.get("fill_opacity")
if opacity == 0:
return True
if color in ("rgba(0,0,0,0)", "rgba(0, 0, 0, 0)"):
return True
return False
def text_payloads_with_numeric_value(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
numeric_payloads: list[dict[str, Any]] = []
for payload in payloads:
text_value = text_or_none(payload.get("text_value"))
if text_value is None:
continue
if any(char.isdigit() for char in text_value):
numeric_payloads.append(payload)
return numeric_payloads
def classify_annotation_issue(item: dict[str, Any]) -> str | None:
original_components = item["original_components"]
engineering_components = item["engineering_components"]
original_has_text = has_component(original_components, "text")
engineering_has_text = has_component(engineering_components, "text")
original_has_icon = has_component(original_components, "icon")
engineering_has_icon = has_component(engineering_components, "icon")
if original_has_text and not engineering_has_text:
return "text_missing_in_engineering"
if not original_has_text and engineering_has_text:
return "extra_text_in_engineering"
if original_has_icon and not engineering_has_icon:
return "icon_missing_in_engineering"
if not original_has_icon and engineering_has_icon:
return "extra_icon_in_engineering"
return None
def classify_style_semantic_issue(item: dict[str, Any]) -> str | None:
source_layer = item["source_layer"] or "unknown"
original_payloads = item.get("original_payloads", {})
engineering_payloads = item.get("engineering_payloads", {})
original_fill_payloads = get_component_payloads(original_payloads, "fill")
engineering_fill_payloads = get_component_payloads(engineering_payloads, "fill")
original_text_payloads = get_component_payloads(original_payloads, "text")
engineering_text_payloads = get_component_payloads(engineering_payloads, "text")
original_icon_payloads = get_component_payloads(original_payloads, "icon")
engineering_icon_payloads = get_component_payloads(engineering_payloads, "icon")
if source_layer in SUPPORT_FILL_SOURCE_LAYERS:
if payload_has_transparent_fill(original_fill_payloads) and payload_has_nontransparent_fill(engineering_fill_payloads):
return "support_layer_strengthened_fill"
original_numeric_texts = text_payloads_with_numeric_value(original_text_payloads)
engineering_numeric_texts = text_payloads_with_numeric_value(engineering_text_payloads)
if source_layer in DEPTH_NUMERIC_SOURCE_LAYERS and original_numeric_texts and not engineering_numeric_texts:
return "depth_numeric_missing"
if source_layer in CLEARANCE_NUMERIC_SOURCE_LAYERS and original_numeric_texts and not engineering_numeric_texts:
return "clearance_numeric_missing"
if source_layer in SAFETY_ICON_SOURCE_LAYERS and original_icon_payloads and not engineering_icon_payloads:
return "safety_icon_missing"
return None
def write_reports(
*,
original_count: int,
@@ -658,6 +857,10 @@ def write_reports(
status_counter: Counter[str],
source_layer_counter: Counter[tuple[str, str]],
mismatch_examples: list[dict[str, Any]],
annotation_issue_counter: Counter[tuple[str, str]],
annotation_issue_examples: list[dict[str, Any]],
style_semantic_issue_counter: Counter[tuple[str, str]],
style_semantic_issue_examples: list[dict[str, Any]],
) -> None:
payload = {
"original_feature_instances": original_count,
@@ -668,21 +871,31 @@ def write_reports(
{"status": status, "source_layer": source_layer, "count": count}
for (status, source_layer), count in source_layer_counter.most_common(30)
],
"annotation_issue_counts": [
{"issue": issue, "source_layer": source_layer, "count": count}
for (issue, source_layer), count in annotation_issue_counter.most_common(30)
],
"style_semantic_issue_counts": [
{"issue": issue, "source_layer": source_layer, "count": count}
for (issue, source_layer), count in style_semantic_issue_counter.most_common(30)
],
"mismatch_examples": mismatch_examples,
"annotation_issue_examples": annotation_issue_examples,
"style_semantic_issue_examples": style_semantic_issue_examples,
}
REPORT_JSON_PATH.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
lines = [
"# NavSea 原始版 vs 工程版渲染审计报告",
f"# NavSea 原始版 vs {COMPARISON_LABEL}渲染审计报告",
"",
"## 范围",
"",
f"- 原始样式: `{ORIGINAL_STYLE_PATH}`",
f"- 工程样式: `{ENGINEERING_STYLE_PATH}`",
f"- {COMPARISON_LABEL}样式: `{ENGINEERING_STYLE_PATH}`",
f"- 原始瓦片根目录: `{ORIGINAL_TILE_ROOT}`",
f"- 工程瓦片根目录: `{ENGINEERING_TILE_ROOT}`",
f"- {COMPARISON_LABEL}瓦片根目录: `{ENGINEERING_TILE_ROOT}`",
f"- 原始 feature 实例数: `{original_count}`",
f"- 工程 feature 实例数: `{engineering_count}`",
f"- {COMPARISON_LABEL} feature 实例数: `{engineering_count}`",
f"- 审计结果数: `{result_count}`",
"",
"## 匹配口径",
@@ -708,6 +921,28 @@ def write_reports(
for (status, source_layer), count in source_layer_counter.most_common(20):
lines.append(f"- `{status}` | `{source_layer}` | `{count}`")
lines.extend([
"",
"## 标注专项问题",
"",
])
if not annotation_issue_counter:
lines.append("- 没有发现标注专项问题。")
else:
for (issue, source_layer), count in annotation_issue_counter.most_common(20):
lines.append(f"- `{issue}` | `{source_layer}` | `{count}`")
lines.extend([
"",
"## Style 语义专项问题",
"",
])
if not style_semantic_issue_counter:
lines.append("- 没有发现 style 语义专项问题。")
else:
for (issue, source_layer), count in style_semantic_issue_counter.most_common(20):
lines.append(f"- `{issue}` | `{source_layer}` | `{count}`")
lines.extend([
"",
"## 差异样例",
@@ -724,6 +959,38 @@ def write_reports(
lines.append(f" original: {format_component_map(item['original_components'])}")
lines.append(f" engineering: {format_component_map(item['engineering_components'])}")
lines.extend([
"",
"## 标注问题样例",
"",
])
if not annotation_issue_examples:
lines.append("- 没有标注问题样例。")
else:
for item in annotation_issue_examples:
lines.append(
f"- `{item['annotation_issue']}` | tile=`{item['tile']}` | source_layer=`{item['source_layer']}` | "
f"fid=`{item['fid_legacy']}` | object=`{item['canonical_object_type'] or 'n/a'}`"
)
lines.append(f" original: {format_component_map(item['original_components'])}")
lines.append(f" engineering: {format_component_map(item['engineering_components'])}")
lines.extend([
"",
"## Style 语义问题样例",
"",
])
if not style_semantic_issue_examples:
lines.append("- 没有 style 语义问题样例。")
else:
for item in style_semantic_issue_examples:
lines.append(
f"- `{item['style_semantic_issue']}` | tile=`{item['tile']}` | source_layer=`{item['source_layer']}` | "
f"fid=`{item['fid_legacy']}` | object=`{item['canonical_object_type'] or 'n/a'}`"
)
lines.append(f" original: {format_component_map(item['original_components'])}")
lines.append(f" engineering: {format_component_map(item['engineering_components'])}")
REPORT_MD_PATH.write_text("\n".join(lines) + "\n", encoding="utf-8")
@@ -865,12 +1132,16 @@ def persist_audit_run(
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Audit original vs engineering render outputs.")
parser.add_argument("--audit-name", default=DEFAULT_AUDIT_NAME)
parser.add_argument("--comparison-label", default="工程版")
parser.add_argument("--fid-key")
parser.add_argument("--match-on-fid-only", action="store_true")
parser.add_argument("--original-style", type=Path, default=DEFAULT_ORIGINAL_STYLE_PATH)
parser.add_argument("--engineering-style", type=Path, default=DEFAULT_ENGINEERING_STYLE_PATH)
parser.add_argument("--original-tile-root", type=Path, default=DEFAULT_ORIGINAL_TILE_ROOT)
parser.add_argument("--engineering-tile-root", type=Path, default=DEFAULT_ENGINEERING_TILE_ROOT)
parser.add_argument("--report-md", type=Path, default=DEFAULT_REPORT_MD_PATH)
parser.add_argument("--report-json", type=Path, default=DEFAULT_REPORT_JSON_PATH)
parser.add_argument("--skip-db-persist", action="store_true")
return parser.parse_args()
@@ -882,15 +1153,23 @@ def main() -> None:
global ENGINEERING_TILE_ROOT
global REPORT_MD_PATH
global REPORT_JSON_PATH
global SKIP_DB_PERSIST
global COMPARISON_LABEL
global FID_CODEC
global MATCH_ON_FID_ONLY
args = parse_args()
AUDIT_NAME = args.audit_name
COMPARISON_LABEL = args.comparison_label
FID_CODEC = NavSeaFidCodec(args.fid_key.encode("utf-8")) if args.fid_key else None
MATCH_ON_FID_ONLY = args.match_on_fid_only
ORIGINAL_STYLE_PATH = args.original_style
ENGINEERING_STYLE_PATH = args.engineering_style
ORIGINAL_TILE_ROOT = args.original_tile_root
ENGINEERING_TILE_ROOT = args.engineering_tile_root
REPORT_MD_PATH = args.report_md
REPORT_JSON_PATH = args.report_json
SKIP_DB_PERSIST = args.skip_db_persist
original_style = load_style(ORIGINAL_STYLE_PATH)
engineering_style = load_style(ENGINEERING_STYLE_PATH)
@@ -902,70 +1181,142 @@ def main() -> None:
status_counter: Counter[str] = Counter()
source_layer_counter: Counter[tuple[str, str]] = Counter()
mismatch_examples: list[dict[str, Any]] = []
annotation_issue_counter: Counter[tuple[str, str]] = Counter()
annotation_issue_examples: list[dict[str, Any]] = []
style_semantic_issue_counter: Counter[tuple[str, str]] = Counter()
style_semantic_issue_examples: list[dict[str, Any]] = []
with db_connect() as conn:
with conn.cursor() as cur:
ensure_audit_tables(cur)
cur.execute("DELETE FROM navsea_render_audit_result")
cur.execute("DELETE FROM navsea_render_audit_run")
conn.commit()
if SKIP_DB_PERSIST:
for engineering_tile in engineering_tiles:
rel = engineering_tile.relative_to(ENGINEERING_TILE_ROOT)
original_tile = ORIGINAL_TILE_ROOT / rel
if not original_tile.exists():
continue
rows: list[tuple[Any, ...]] = []
for engineering_tile in engineering_tiles:
rel = engineering_tile.relative_to(ENGINEERING_TILE_ROOT)
original_tile = ORIGINAL_TILE_ROOT / rel
if not original_tile.exists():
continue
original_instances = decode_tile_instances("original", ORIGINAL_TILE_ROOT, original_style, original_tile)
engineering_instances = decode_tile_instances(
"engineering", ENGINEERING_TILE_ROOT, engineering_style, engineering_tile
)
original_count += len(original_instances)
engineering_count += len(engineering_instances)
all_instance_ids = sorted(set(original_instances) | set(engineering_instances))
for instance_id in all_instance_ids:
item = compare_instance(original_instances.get(instance_id), engineering_instances.get(instance_id))
result_count += 1
status_counter[item["status"]] += 1
if item["status"] != "exact_match":
source_layer_counter[(item["status"], item["source_layer"] or "unknown")] += 1
if len(mismatch_examples) < MAX_EXAMPLES:
mismatch_examples.append(item)
rows.append(
(
AUDIT_NAME,
hash_text(item["object_instance_id"]),
item["object_instance_id"],
item["object_id"],
item["fid_legacy"],
item["tile_z"],
item["tile_x"],
item["tile_y"],
item["source_layer"],
item["canonical_object_type"],
item["status"],
json.dumps(item["original_components"], ensure_ascii=False),
json.dumps(item["engineering_components"], ensure_ascii=False),
)
)
if len(rows) >= DB_BATCH_SIZE:
insert_result_batch(cur, rows)
conn.commit()
rows.clear()
insert_result_batch(cur, rows)
persist_audit_run(
cur,
original_count=original_count,
engineering_count=engineering_count,
result_count=result_count,
status_counter=status_counter,
source_layer_counter=source_layer_counter,
original_instances = decode_tile_instances("original", ORIGINAL_TILE_ROOT, original_style, original_tile)
engineering_instances = decode_tile_instances(
"engineering", ENGINEERING_TILE_ROOT, engineering_style, engineering_tile
)
conn.commit()
original_count += len(original_instances)
engineering_count += len(engineering_instances)
all_instance_ids = sorted(set(original_instances) | set(engineering_instances))
for instance_id in all_instance_ids:
item = compare_instance(original_instances.get(instance_id), engineering_instances.get(instance_id))
result_count += 1
status_counter[item["status"]] += 1
if item["status"] != "exact_match":
source_layer_counter[(item["status"], item["source_layer"] or "unknown")] += 1
if len(mismatch_examples) < MAX_EXAMPLES:
mismatch_examples.append(item)
annotation_issue = classify_annotation_issue(item)
if annotation_issue:
source_layer = item["source_layer"] or "unknown"
annotation_issue_counter[(annotation_issue, source_layer)] += 1
if len(annotation_issue_examples) < MAX_EXAMPLES:
annotation_issue_examples.append(
{
**item,
"annotation_issue": annotation_issue,
}
)
style_semantic_issue = classify_style_semantic_issue(item)
if style_semantic_issue:
source_layer = item["source_layer"] or "unknown"
style_semantic_issue_counter[(style_semantic_issue, source_layer)] += 1
if len(style_semantic_issue_examples) < MAX_EXAMPLES:
style_semantic_issue_examples.append(
{
**item,
"style_semantic_issue": style_semantic_issue,
}
)
else:
with db_connect() as conn:
with conn.cursor() as cur:
ensure_audit_tables(cur)
cur.execute("TRUNCATE TABLE navsea_render_audit_result")
cur.execute("TRUNCATE TABLE navsea_render_audit_run")
conn.commit()
rows: list[tuple[Any, ...]] = []
for engineering_tile in engineering_tiles:
rel = engineering_tile.relative_to(ENGINEERING_TILE_ROOT)
original_tile = ORIGINAL_TILE_ROOT / rel
if not original_tile.exists():
continue
original_instances = decode_tile_instances("original", ORIGINAL_TILE_ROOT, original_style, original_tile)
engineering_instances = decode_tile_instances(
"engineering", ENGINEERING_TILE_ROOT, engineering_style, engineering_tile
)
original_count += len(original_instances)
engineering_count += len(engineering_instances)
all_instance_ids = sorted(set(original_instances) | set(engineering_instances))
for instance_id in all_instance_ids:
item = compare_instance(original_instances.get(instance_id), engineering_instances.get(instance_id))
result_count += 1
status_counter[item["status"]] += 1
if item["status"] != "exact_match":
source_layer_counter[(item["status"], item["source_layer"] or "unknown")] += 1
if len(mismatch_examples) < MAX_EXAMPLES:
mismatch_examples.append(item)
annotation_issue = classify_annotation_issue(item)
if annotation_issue:
source_layer = item["source_layer"] or "unknown"
annotation_issue_counter[(annotation_issue, source_layer)] += 1
if len(annotation_issue_examples) < MAX_EXAMPLES:
annotation_issue_examples.append(
{
**item,
"annotation_issue": annotation_issue,
}
)
style_semantic_issue = classify_style_semantic_issue(item)
if style_semantic_issue:
source_layer = item["source_layer"] or "unknown"
style_semantic_issue_counter[(style_semantic_issue, source_layer)] += 1
if len(style_semantic_issue_examples) < MAX_EXAMPLES:
style_semantic_issue_examples.append(
{
**item,
"style_semantic_issue": style_semantic_issue,
}
)
rows.append(
(
AUDIT_NAME,
hash_text(item["object_instance_id"]),
item["object_instance_id"],
item["object_id"],
item["fid_legacy"],
item["tile_z"],
item["tile_x"],
item["tile_y"],
item["source_layer"],
item["canonical_object_type"],
item["status"],
json.dumps(item["original_components"], ensure_ascii=False),
json.dumps(item["engineering_components"], ensure_ascii=False),
)
)
if len(rows) >= DB_BATCH_SIZE:
insert_result_batch(cur, rows)
conn.commit()
rows.clear()
insert_result_batch(cur, rows)
persist_audit_run(
cur,
original_count=original_count,
engineering_count=engineering_count,
result_count=result_count,
status_counter=status_counter,
source_layer_counter=source_layer_counter,
)
conn.commit()
write_reports(
original_count=original_count,
@@ -974,6 +1325,10 @@ def main() -> None:
status_counter=status_counter,
source_layer_counter=source_layer_counter,
mismatch_examples=mismatch_examples,
annotation_issue_counter=annotation_issue_counter,
annotation_issue_examples=annotation_issue_examples,
style_semantic_issue_counter=style_semantic_issue_counter,
style_semantic_issue_examples=style_semantic_issue_examples,
)
print(
json.dumps(