1009 lines
31 KiB
Python
1009 lines
31 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
r"""
|
||
CM95 / CM93 MySQL 选定 AOI → NavSea PBF 映射输入审计 v2
|
||
|
||
用途
|
||
====
|
||
从已经通过最终验收的 CM95/CM93 MySQL Dataset 中，只读导出选定 AOI
|
||
（默认对应现有 NavSea 样本 tile z10/914/385）内的真实对象与属性分布，
|
||
作为下一步 `cm95_to_navsea_v1.yaml` 的唯一 MySQL 输入依据。
|
||
|
||
默认 AOI：
|
||
z10 / x=914 / y=385
|
||
lon: 129.7265625 .. 130.078125
|
||
lat: 33.43144133557529 .. 33.7243396617476
|
||
|
||
该 z10 tile 已覆盖本次提供的：
|
||
z11/1828-1829/770-771
|
||
z12/3656-3659/1540-1543
|
||
|
||
本脚本：
|
||
- 只读 MySQL；
|
||
- 不修改 importer/schema；
|
||
- 不生成 PBF；
|
||
- 不依赖历史 V 系列结果；
|
||
- 不需要 NEWPEC/旧 PBF 数据库；
|
||
- 不涉及 style 修改。
|
||
|
||
输出：
|
||
- AOI报告_v1.json
|
||
- 对象类别汇总_v1.csv
|
||
- 对象类别尺度分布_v1.csv
|
||
- 属性字段汇总_v1.csv
|
||
- 属性值域汇总_v1.csv
|
||
- 对象字典_v1.csv
|
||
- 属性字典_v1.csv
|
||
- AOI_Cell清单_v1.csv
|
||
- AOI_Feature样本_v1.jsonl
|
||
- AOI_关联对象样本_v1.jsonl
|
||
- AOI_对象映射准备表_v1.csv
|
||
- 摘要_v1.txt
|
||
- ZIP
|
||
|
||
执行示例：
|
||
python .\cm95_karatsu_aoi_mapping_audit_v1.py `
|
||
--host 127.0.0.1 `
|
||
--port 3306 `
|
||
--user root `
|
||
--database cm93_chart `
|
||
--dataset-id 1
|
||
|
||
如不传密码，会安全提示输入。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import getpass
|
||
import json
|
||
import math
|
||
import sys
|
||
import zipfile
|
||
from collections import Counter, defaultdict
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
|
||
try:
|
||
import pymysql
|
||
from pymysql.cursors import DictCursor
|
||
except ImportError:
|
||
print("缺少 pymysql，请执行: pip install pymysql")
|
||
sys.exit(2)
|
||
|
||
|
||
VERSION = "v2.0"
|
||
|
||
|
||
def parse_args():
|
||
p = argparse.ArgumentParser(
|
||
description="CM95 选定 AOI → NavSea PBF 映射输入审计 v2"
|
||
)
|
||
p.add_argument("--host", default="127.0.0.1")
|
||
p.add_argument("--port", type=int, default=3306)
|
||
p.add_argument("--user", default="root")
|
||
p.add_argument("--password", default=None)
|
||
p.add_argument("--database", default="cm93_chart")
|
||
p.add_argument("--dataset-id", type=int, default=1)
|
||
|
||
p.add_argument("--z", type=int, default=10)
|
||
p.add_argument("--x", type=int, default=914)
|
||
p.add_argument("--y", type=int, default=385)
|
||
|
||
p.add_argument(
|
||
"--sample-per-class",
|
||
type=int,
|
||
default=12,
|
||
help="每个 object_class + geometry_type 最多保存多少条 Feature 样本",
|
||
)
|
||
p.add_argument(
|
||
"--value-sample-limit",
|
||
type=int,
|
||
default=40,
|
||
help="每个属性值域最多在 CSV 中列出多少个高频值",
|
||
)
|
||
p.add_argument("--output-dir", default=None)
|
||
return p.parse_args()
|
||
|
||
|
||
def tile_bounds(z: int, x: int, y: int):
|
||
n = 2 ** z
|
||
lon_min = x / n * 360.0 - 180.0
|
||
lon_max = (x + 1) / n * 360.0 - 180.0
|
||
|
||
def tile_lat(ty: int) -> float:
|
||
return math.degrees(
|
||
math.atan(
|
||
math.sinh(
|
||
math.pi * (1.0 - 2.0 * ty / n)
|
||
)
|
||
)
|
||
)
|
||
|
||
lat_max = tile_lat(y)
|
||
lat_min = tile_lat(y + 1)
|
||
return lon_min, lat_min, lon_max, lat_max
|
||
|
||
|
||
def begin_read_only(cur, warnings: list[str]):
|
||
try:
|
||
cur.execute("SET SESSION TRANSACTION READ ONLY")
|
||
except Exception as e:
|
||
warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}")
|
||
|
||
try:
|
||
cur.execute("START TRANSACTION READ ONLY")
|
||
except Exception as e:
|
||
warnings.append(f"START TRANSACTION READ ONLY failed: {e}")
|
||
cur.execute("START TRANSACTION")
|
||
|
||
|
||
def scalar(cur, sql: str, params=()):
|
||
cur.execute(sql, params)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
return next(iter(row.values()))
|
||
|
||
|
||
def fetchall(cur, sql: str, params=()):
|
||
cur.execute(sql, params)
|
||
return list(cur.fetchall())
|
||
|
||
|
||
def write_csv(path: Path, rows: Iterable[dict[str, Any]]):
|
||
rows = list(rows)
|
||
if not rows:
|
||
path.write_text("", encoding="utf-8-sig")
|
||
return
|
||
|
||
keys = []
|
||
seen = set()
|
||
for row in rows:
|
||
for key in row:
|
||
if key not in seen:
|
||
seen.add(key)
|
||
keys.append(key)
|
||
|
||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||
w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore")
|
||
w.writeheader()
|
||
for row in rows:
|
||
out = {}
|
||
for key in keys:
|
||
value = row.get(key)
|
||
if isinstance(value, (dict, list, tuple)):
|
||
value = json.dumps(
|
||
value,
|
||
ensure_ascii=False,
|
||
default=str,
|
||
)
|
||
out[key] = value
|
||
w.writerow(out)
|
||
|
||
|
||
def json_obj(v):
|
||
if v is None:
|
||
return {}
|
||
if isinstance(v, dict):
|
||
return v
|
||
if isinstance(v, (bytes, bytearray, memoryview)):
|
||
v = bytes(v).decode("utf-8")
|
||
obj = json.loads(v)
|
||
return obj if isinstance(obj, dict) else {}
|
||
|
||
|
||
def normalized_attr_value(row: dict[str, Any]):
|
||
vt = row.get("value_type")
|
||
if vt in ("BYTE", "INT", "LONG"):
|
||
return row.get("value_int")
|
||
if vt in ("REAL", "WORD10"):
|
||
return row.get("value_real")
|
||
if vt == "STRING":
|
||
return row.get("value_text")
|
||
if vt in ("LIST", "COMPLEX"):
|
||
value = row.get("value_json")
|
||
if value is not None:
|
||
try:
|
||
if isinstance(value, (dict, list)):
|
||
return value
|
||
return json.loads(value)
|
||
except Exception:
|
||
return str(value)
|
||
return row.get("value_text")
|
||
return (
|
||
row.get("value_text")
|
||
if row.get("value_text") is not None
|
||
else row.get("value_int")
|
||
if row.get("value_int") is not None
|
||
else row.get("value_real")
|
||
)
|
||
|
||
|
||
def stable_value_text(value: Any) -> str:
|
||
if isinstance(value, (dict, list)):
|
||
return json.dumps(
|
||
value,
|
||
ensure_ascii=False,
|
||
sort_keys=True,
|
||
separators=(",", ":"),
|
||
default=str,
|
||
)
|
||
if value is None:
|
||
return "<NULL>"
|
||
if isinstance(value, float):
|
||
return format(value, ".15g")
|
||
return str(value)
|
||
|
||
|
||
def suggested_source_id(
|
||
dataset_id: int,
|
||
cell_path: str,
|
||
source_record_index: int,
|
||
geometry_type: str,
|
||
):
|
||
# 这里只输出建议的稳定 identity 文本，不在本阶段分配 int32 fid。
|
||
return (
|
||
f"cm95:{dataset_id}:"
|
||
f"{cell_path.replace(chr(92), '/')}:"
|
||
f"{source_record_index}:"
|
||
f"{geometry_type.lower()}"
|
||
)
|
||
|
||
|
||
def main():
|
||
a = parse_args()
|
||
|
||
if a.password is None:
|
||
a.password = getpass.getpass("MySQL password: ")
|
||
|
||
lon_min, lat_min, lon_max, lat_max = tile_bounds(a.z, a.x, a.y)
|
||
|
||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
outdir = Path(
|
||
a.output_dir
|
||
or f"cm95_selected_aoi_mapping_audit_dataset_{a.dataset_id}_{stamp}"
|
||
).resolve()
|
||
outdir.mkdir(parents=True, exist_ok=True)
|
||
|
||
warnings: list[str] = []
|
||
|
||
conn = pymysql.connect(
|
||
host=a.host,
|
||
port=a.port,
|
||
user=a.user,
|
||
password=a.password,
|
||
database=a.database,
|
||
charset="utf8mb4",
|
||
cursorclass=DictCursor,
|
||
autocommit=False,
|
||
connect_timeout=20,
|
||
read_timeout=900,
|
||
write_timeout=30,
|
||
)
|
||
|
||
report: dict[str, Any] = {
|
||
"version": "v2.0",
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"database": a.database,
|
||
"dataset_id": a.dataset_id,
|
||
"read_only": True,
|
||
"aoi": {
|
||
"tile": {
|
||
"z": a.z,
|
||
"x": a.x,
|
||
"y": a.y,
|
||
},
|
||
"bbox_wgs84": {
|
||
"min_lon": lon_min,
|
||
"min_lat": lat_min,
|
||
"max_lon": lon_max,
|
||
"max_lat": lat_max,
|
||
},
|
||
"known_navsea_samples_inside": [
|
||
"z10/914/385",
|
||
"z11/1828-1829/770-771",
|
||
"z12/3656-3659/1540-1543",
|
||
],
|
||
},
|
||
}
|
||
|
||
bbox_where = """
|
||
f.dataset_id=%s
|
||
AND f.bbox_min_lon IS NOT NULL
|
||
AND f.bbox_min_lat IS NOT NULL
|
||
AND f.bbox_max_lon IS NOT NULL
|
||
AND f.bbox_max_lat IS NOT NULL
|
||
AND f.bbox_max_lon >= %s
|
||
AND f.bbox_min_lon <= %s
|
||
AND f.bbox_max_lat >= %s
|
||
AND f.bbox_min_lat <= %s
|
||
"""
|
||
bbox_params = (
|
||
a.dataset_id,
|
||
lon_min,
|
||
lon_max,
|
||
lat_min,
|
||
lat_max,
|
||
)
|
||
|
||
try:
|
||
with conn.cursor() as cur:
|
||
begin_read_only(cur, warnings)
|
||
|
||
print("[1/8] Dataset / AOI Cell ...")
|
||
|
||
dataset = fetchall(
|
||
cur,
|
||
"""
|
||
SELECT
|
||
id,
|
||
source_type,
|
||
name,
|
||
source_root,
|
||
parser_version,
|
||
normalized_srid
|
||
FROM chart_dataset
|
||
WHERE id=%s
|
||
""",
|
||
(a.dataset_id,),
|
||
)
|
||
if not dataset:
|
||
raise RuntimeError(
|
||
f"Dataset ID={a.dataset_id} 不存在"
|
||
)
|
||
report["dataset"] = dataset[0]
|
||
|
||
cells = fetchall(
|
||
cur,
|
||
"""
|
||
SELECT
|
||
id AS cell_id,
|
||
source_relative_path,
|
||
source_file_name,
|
||
scale_code,
|
||
native_scale,
|
||
cell_name,
|
||
feature_count,
|
||
attribute_count,
|
||
bbox_min_lon,
|
||
bbox_min_lat,
|
||
bbox_max_lon,
|
||
bbox_max_lat
|
||
FROM chart_cell
|
||
WHERE dataset_id=%s
|
||
AND bbox_max_lon >= %s
|
||
AND bbox_min_lon <= %s
|
||
AND bbox_max_lat >= %s
|
||
AND bbox_min_lat <= %s
|
||
ORDER BY native_scale,source_relative_path
|
||
""",
|
||
(
|
||
a.dataset_id,
|
||
lon_min,
|
||
lon_max,
|
||
lat_min,
|
||
lat_max,
|
||
),
|
||
)
|
||
write_csv(outdir / "AOI_Cell清单_v1.csv", cells)
|
||
|
||
print("[2/8] AOI Feature 对象类别统计 ...")
|
||
|
||
class_summary = fetchall(
|
||
cur,
|
||
f"""
|
||
SELECT
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type,
|
||
COUNT(*) AS feature_count,
|
||
COUNT(DISTINCT f.cell_id) AS cell_count,
|
||
MIN(f.native_scale) AS min_native_scale,
|
||
MAX(f.native_scale) AS max_native_scale,
|
||
SUM(CASE WHEN f.geometry_repaired=1 THEN 1 ELSE 0 END)
|
||
AS repaired_count,
|
||
SUM(CASE WHEN f.geometry_issue IS NOT NULL THEN 1 ELSE 0 END)
|
||
AS issue_count
|
||
FROM chart_feature f
|
||
WHERE {bbox_where}
|
||
GROUP BY
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type
|
||
ORDER BY
|
||
feature_count DESC,
|
||
f.object_class,
|
||
f.geometry_type
|
||
""",
|
||
bbox_params,
|
||
)
|
||
|
||
object_dict = {
|
||
int(r["object_class_id"]): r
|
||
for r in fetchall(
|
||
cur,
|
||
"""
|
||
SELECT
|
||
object_class_id,
|
||
object_class,
|
||
geometry_code,
|
||
display_code,
|
||
description,
|
||
raw_line
|
||
FROM chart_dictionary_object
|
||
WHERE dataset_id=%s
|
||
""",
|
||
(a.dataset_id,),
|
||
)
|
||
}
|
||
|
||
for row in class_summary:
|
||
oid = row.get("object_class_id")
|
||
d = object_dict.get(int(oid)) if oid is not None else None
|
||
row["dictionary_geometry_code"] = (
|
||
d.get("geometry_code") if d else None
|
||
)
|
||
row["dictionary_display_code"] = (
|
||
d.get("display_code") if d else None
|
||
)
|
||
row["description"] = (
|
||
d.get("description") if d else None
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "对象类别汇总_v1.csv",
|
||
class_summary,
|
||
)
|
||
|
||
scale_summary = fetchall(
|
||
cur,
|
||
f"""
|
||
SELECT
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.scale_code,
|
||
f.native_scale,
|
||
COUNT(*) AS feature_count
|
||
FROM chart_feature f
|
||
WHERE {bbox_where}
|
||
GROUP BY
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.scale_code,
|
||
f.native_scale
|
||
ORDER BY
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.native_scale
|
||
""",
|
||
bbox_params,
|
||
)
|
||
write_csv(
|
||
outdir / "对象类别尺度分布_v1.csv",
|
||
scale_summary,
|
||
)
|
||
|
||
print("[3/8] 属性字段 / 值域统计 ...")
|
||
|
||
attr_rows = fetchall(
|
||
cur,
|
||
f"""
|
||
SELECT
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.native_scale,
|
||
a.attribute_id,
|
||
a.attribute_name,
|
||
a.value_type,
|
||
a.value_int,
|
||
a.value_real,
|
||
a.value_text,
|
||
a.value_json
|
||
FROM chart_feature_attribute a
|
||
JOIN chart_feature f ON f.id=a.feature_id
|
||
WHERE {bbox_where}
|
||
ORDER BY
|
||
f.object_class,
|
||
f.geometry_type,
|
||
a.attribute_name
|
||
""",
|
||
bbox_params,
|
||
)
|
||
|
||
attr_stats: dict[
|
||
tuple[Any, ...],
|
||
dict[str, Any],
|
||
] = {}
|
||
attr_value_counts: dict[
|
||
tuple[Any, ...],
|
||
Counter[str],
|
||
] = defaultdict(Counter)
|
||
attr_scale_values: dict[
|
||
tuple[Any, ...],
|
||
set[int],
|
||
] = defaultdict(set)
|
||
|
||
for row in attr_rows:
|
||
key = (
|
||
row["object_class_id"],
|
||
row["object_class"],
|
||
row["geometry_type"],
|
||
row["attribute_id"],
|
||
row["attribute_name"],
|
||
row["value_type"],
|
||
)
|
||
if key not in attr_stats:
|
||
attr_stats[key] = {
|
||
"object_class_id": row["object_class_id"],
|
||
"object_class": row["object_class"],
|
||
"geometry_type": row["geometry_type"],
|
||
"attribute_id": row["attribute_id"],
|
||
"attribute_name": row["attribute_name"],
|
||
"value_type": row["value_type"],
|
||
"occurrence_count": 0,
|
||
}
|
||
|
||
attr_stats[key]["occurrence_count"] += 1
|
||
attr_scale_values[key].add(int(row["native_scale"]))
|
||
|
||
value = normalized_attr_value(row)
|
||
attr_value_counts[key][stable_value_text(value)] += 1
|
||
|
||
attr_summary_rows = []
|
||
attr_value_rows = []
|
||
|
||
for key, info in attr_stats.items():
|
||
values = attr_value_counts[key]
|
||
info = dict(info)
|
||
info["distinct_value_count"] = len(values)
|
||
info["native_scales"] = sorted(attr_scale_values[key])
|
||
info["top_values"] = [
|
||
{
|
||
"value": value,
|
||
"count": count,
|
||
}
|
||
for value, count
|
||
in values.most_common(a.value_sample_limit)
|
||
]
|
||
attr_summary_rows.append(info)
|
||
|
||
for rank, (value, count) in enumerate(
|
||
values.most_common(a.value_sample_limit),
|
||
start=1,
|
||
):
|
||
attr_value_rows.append({
|
||
"object_class_id": key[0],
|
||
"object_class": key[1],
|
||
"geometry_type": key[2],
|
||
"attribute_id": key[3],
|
||
"attribute_name": key[4],
|
||
"value_type": key[5],
|
||
"rank": rank,
|
||
"value": value,
|
||
"count": count,
|
||
"total_distinct_values": len(values),
|
||
})
|
||
|
||
attr_summary_rows.sort(
|
||
key=lambda r: (
|
||
str(r["object_class"]),
|
||
str(r["geometry_type"]),
|
||
str(r["attribute_name"]),
|
||
)
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "属性字段汇总_v1.csv",
|
||
attr_summary_rows,
|
||
)
|
||
write_csv(
|
||
outdir / "属性值域汇总_v1.csv",
|
||
attr_value_rows,
|
||
)
|
||
|
||
print("[4/8] Feature 样本 / attributes_json ...")
|
||
|
||
sample_rows = fetchall(
|
||
cur,
|
||
f"""
|
||
SELECT
|
||
f.id AS feature_id,
|
||
f.cell_id,
|
||
c.source_relative_path,
|
||
f.source_record_index,
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.scale_code,
|
||
f.native_scale,
|
||
f.raw_geotype,
|
||
f.coverage_id,
|
||
f.bbox_min_lon,
|
||
f.bbox_min_lat,
|
||
f.bbox_max_lon,
|
||
f.bbox_max_lat,
|
||
f.attributes_json,
|
||
f.geometry_repaired,
|
||
f.geometry_issue
|
||
FROM chart_feature f
|
||
JOIN chart_cell c ON c.id=f.cell_id
|
||
WHERE {bbox_where}
|
||
ORDER BY
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.native_scale,
|
||
c.source_relative_path,
|
||
f.source_record_index
|
||
""",
|
||
bbox_params,
|
||
)
|
||
|
||
class_seen: Counter[tuple[str, str]] = Counter()
|
||
selected_samples: list[dict[str, Any]] = []
|
||
selected_feature_ids: set[int] = set()
|
||
|
||
for row in sample_rows:
|
||
key = (
|
||
str(row["object_class"]),
|
||
str(row["geometry_type"]),
|
||
)
|
||
if class_seen[key] >= a.sample_per_class:
|
||
continue
|
||
|
||
class_seen[key] += 1
|
||
|
||
attrs = json_obj(row.pop("attributes_json"))
|
||
row["attributes"] = attrs
|
||
row["suggested_source_id"] = suggested_source_id(
|
||
a.dataset_id,
|
||
str(row["source_relative_path"]),
|
||
int(row["source_record_index"]),
|
||
str(row["geometry_type"]),
|
||
)
|
||
selected_samples.append(row)
|
||
selected_feature_ids.add(int(row["feature_id"]))
|
||
|
||
with (
|
||
outdir / "AOI_Feature样本_v1.jsonl"
|
||
).open("w", encoding="utf-8") as f:
|
||
for row in selected_samples:
|
||
f.write(
|
||
json.dumps(
|
||
row,
|
||
ensure_ascii=False,
|
||
default=str,
|
||
)
|
||
+ "\n"
|
||
)
|
||
|
||
print("[5/8] 关联对象样本 ...")
|
||
|
||
# 从样本里的 _cm93_related_indices 追踪同 Cell Feature，
|
||
# 仅作为 mapping 调查辅助；不假设关系方向。
|
||
related_keys: set[tuple[int, int]] = set()
|
||
|
||
for row in selected_samples:
|
||
attrs = row.get("attributes") or {}
|
||
related = attrs.get("_cm93_related_indices")
|
||
if not isinstance(related, list):
|
||
continue
|
||
|
||
for value in related:
|
||
try:
|
||
related_keys.add(
|
||
(
|
||
int(row["cell_id"]),
|
||
int(value),
|
||
)
|
||
)
|
||
except Exception:
|
||
continue
|
||
|
||
related_rows: list[dict[str, Any]] = []
|
||
|
||
# AOI 样本量有限，按 key 查询，避免构造超长 SQL IN。
|
||
for cell_id, source_index in sorted(related_keys):
|
||
rows = fetchall(
|
||
cur,
|
||
"""
|
||
SELECT
|
||
f.id AS feature_id,
|
||
f.cell_id,
|
||
c.source_relative_path,
|
||
f.source_record_index,
|
||
f.object_class_id,
|
||
f.object_class,
|
||
f.geometry_type,
|
||
f.scale_code,
|
||
f.native_scale,
|
||
f.raw_geotype,
|
||
f.attributes_json
|
||
FROM chart_feature f
|
||
JOIN chart_cell c ON c.id=f.cell_id
|
||
WHERE f.dataset_id=%s
|
||
AND f.cell_id=%s
|
||
AND f.source_record_index=%s
|
||
LIMIT 1
|
||
""",
|
||
(
|
||
a.dataset_id,
|
||
cell_id,
|
||
source_index,
|
||
),
|
||
)
|
||
if not rows:
|
||
continue
|
||
row = rows[0]
|
||
row["attributes"] = json_obj(
|
||
row.pop("attributes_json")
|
||
)
|
||
row["suggested_source_id"] = suggested_source_id(
|
||
a.dataset_id,
|
||
str(row["source_relative_path"]),
|
||
int(row["source_record_index"]),
|
||
str(row["geometry_type"]),
|
||
)
|
||
related_rows.append(row)
|
||
|
||
with (
|
||
outdir / "AOI_关联对象样本_v1.jsonl"
|
||
).open("w", encoding="utf-8") as f:
|
||
for row in related_rows:
|
||
f.write(
|
||
json.dumps(
|
||
row,
|
||
ensure_ascii=False,
|
||
default=str,
|
||
)
|
||
+ "\n"
|
||
)
|
||
|
||
print("[6/8] Dictionary 子集 ...")
|
||
|
||
seen_object_ids = {
|
||
int(r["object_class_id"])
|
||
for r in class_summary
|
||
if r.get("object_class_id") is not None
|
||
}
|
||
object_dict_rows = [
|
||
object_dict[oid]
|
||
for oid in sorted(seen_object_ids)
|
||
if oid in object_dict
|
||
]
|
||
write_csv(
|
||
outdir / "对象字典_v1.csv",
|
||
object_dict_rows,
|
||
)
|
||
|
||
seen_attr_ids = {
|
||
int(r["attribute_id"])
|
||
for r in attr_summary_rows
|
||
if r.get("attribute_id") is not None
|
||
}
|
||
|
||
all_attr_dict = fetchall(
|
||
cur,
|
||
"""
|
||
SELECT
|
||
attribute_id,
|
||
attribute_name,
|
||
source_type_name,
|
||
decoder_type,
|
||
min_value,
|
||
max_value,
|
||
raw_line
|
||
FROM chart_dictionary_attribute
|
||
WHERE dataset_id=%s
|
||
ORDER BY attribute_id
|
||
""",
|
||
(a.dataset_id,),
|
||
)
|
||
attr_dict_rows = [
|
||
r
|
||
for r in all_attr_dict
|
||
if int(r["attribute_id"]) in seen_attr_ids
|
||
]
|
||
write_csv(
|
||
outdir / "属性字典_v1.csv",
|
||
attr_dict_rows,
|
||
)
|
||
|
||
print("[7/8] NavSea mapping 准备表 ...")
|
||
|
||
attrs_by_class: dict[
|
||
tuple[Any, Any, Any],
|
||
list[str],
|
||
] = defaultdict(list)
|
||
|
||
for row in attr_summary_rows:
|
||
ckey = (
|
||
row["object_class_id"],
|
||
row["object_class"],
|
||
row["geometry_type"],
|
||
)
|
||
attrs_by_class[ckey].append(
|
||
str(row["attribute_name"])
|
||
)
|
||
|
||
mapping_prepare = []
|
||
|
||
for row in class_summary:
|
||
key = (
|
||
row["object_class_id"],
|
||
row["object_class"],
|
||
row["geometry_type"],
|
||
)
|
||
mapping_prepare.append({
|
||
"cm95_object_class_id": row["object_class_id"],
|
||
"cm95_object_class": row["object_class"],
|
||
"cm95_description": row.get("description"),
|
||
"geometry_type": row["geometry_type"],
|
||
"aoi_feature_count": row["feature_count"],
|
||
"aoi_cell_count": row["cell_count"],
|
||
"min_native_scale": row["min_native_scale"],
|
||
"max_native_scale": row["max_native_scale"],
|
||
"available_attributes": sorted(
|
||
set(attrs_by_class.get(key, []))
|
||
),
|
||
"target_source_layer": "",
|
||
"canonical_family": "",
|
||
"canonical_object_type": "",
|
||
"chart_render_type": "",
|
||
"safety_class": "",
|
||
"mapping_status": "UNMAPPED",
|
||
"notes": "",
|
||
})
|
||
|
||
write_csv(
|
||
outdir / "AOI_对象映射准备表_v1.csv",
|
||
mapping_prepare,
|
||
)
|
||
|
||
print("[8/8] 汇总 ...")
|
||
|
||
spatial_feature_count = int(
|
||
scalar(
|
||
cur,
|
||
f"""
|
||
SELECT COUNT(*)
|
||
FROM chart_feature f
|
||
WHERE {bbox_where}
|
||
""",
|
||
bbox_params,
|
||
)
|
||
or 0
|
||
)
|
||
|
||
geometry_counts = fetchall(
|
||
cur,
|
||
f"""
|
||
SELECT
|
||
f.geometry_type,
|
||
COUNT(*) AS feature_count
|
||
FROM chart_feature f
|
||
WHERE {bbox_where}
|
||
GROUP BY f.geometry_type
|
||
ORDER BY f.geometry_type
|
||
""",
|
||
bbox_params,
|
||
)
|
||
|
||
report.update({
|
||
"warnings": warnings,
|
||
"counts": {
|
||
"aoi_cells": len(cells),
|
||
"aoi_spatial_features": spatial_feature_count,
|
||
"object_class_geometry_groups": len(class_summary),
|
||
"attribute_group_count": len(attr_summary_rows),
|
||
"sample_feature_count": len(selected_samples),
|
||
"related_sample_count": len(related_rows),
|
||
},
|
||
"geometry_counts": geometry_counts,
|
||
"object_classes": [
|
||
{
|
||
"object_class_id": r["object_class_id"],
|
||
"object_class": r["object_class"],
|
||
"geometry_type": r["geometry_type"],
|
||
"feature_count": r["feature_count"],
|
||
"description": r.get("description"),
|
||
}
|
||
for r in class_summary
|
||
],
|
||
"next_step": (
|
||
"Use AOI_对象映射准备表_v1.csv + 属性值域汇总_v1.csv "
|
||
"to author cm95_to_navsea_v1.yaml."
|
||
),
|
||
})
|
||
|
||
conn.rollback()
|
||
|
||
finally:
|
||
conn.close()
|
||
|
||
(
|
||
outdir / "AOI报告_v1.json"
|
||
).write_text(
|
||
json.dumps(
|
||
report,
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
default=str,
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
lines = [
|
||
"CM95 选定 AOI → NavSea PBF 映射输入审计 v2",
|
||
"=" * 84,
|
||
f"Dataset ID : {a.dataset_id}",
|
||
f"Tile : z{a.z}/{a.x}/{a.y}",
|
||
(
|
||
"BBox WGS84 : "
|
||
f"{lon_min:.10f},{lat_min:.10f} "
|
||
f"→ {lon_max:.10f},{lat_max:.10f}"
|
||
),
|
||
"",
|
||
f"AOI cells : {report['counts']['aoi_cells']}",
|
||
f"AOI spatial features : {report['counts']['aoi_spatial_features']}",
|
||
(
|
||
"Object class/geom groups : "
|
||
f"{report['counts']['object_class_geometry_groups']}"
|
||
),
|
||
(
|
||
"Attribute groups : "
|
||
f"{report['counts']['attribute_group_count']}"
|
||
),
|
||
f"Feature samples : {report['counts']['sample_feature_count']}",
|
||
f"Related samples : {report['counts']['related_sample_count']}",
|
||
"",
|
||
"Geometry counts:",
|
||
]
|
||
|
||
for row in report["geometry_counts"]:
|
||
lines.append(
|
||
f" {str(row['geometry_type']):10s} "
|
||
f"{int(row['feature_count'])}"
|
||
)
|
||
|
||
lines += [
|
||
"",
|
||
"NEXT:",
|
||
" 上传本脚本生成的 ZIP。",
|
||
" 下一步直接生成 cm95_to_navsea_v1.yaml 初版并标注 UNMAPPED / hard_gate 缺口。",
|
||
]
|
||
|
||
summary = "\n".join(lines)
|
||
(
|
||
outdir / "摘要_v1.txt"
|
||
).write_text(summary, encoding="utf-8")
|
||
|
||
zip_path = outdir.with_suffix(".zip")
|
||
with zipfile.ZipFile(
|
||
zip_path,
|
||
"w",
|
||
compression=zipfile.ZIP_DEFLATED,
|
||
) as zf:
|
||
for p in sorted(outdir.rglob("*")):
|
||
if p.is_file():
|
||
zf.write(
|
||
p,
|
||
p.relative_to(outdir.parent),
|
||
)
|
||
|
||
print("")
|
||
print(summary)
|
||
print("")
|
||
print(f"报告目录: {outdir}")
|
||
print(f"报告 ZIP : {zip_path}")
|
||
print("把 ZIP 上传给 ChatGPT。")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|