#!/usr/bin/env python3 # -*- coding: utf-8 -*- r""" CM95 / CM93 MySQL 唐津 AOI → NavSea PBF 映射输入审计 v1 用途 ==== 从已经通过最终验收的 CM95/CM93 MySQL Dataset 中,只读导出唐津 AOI (默认对应现有 NavSea 样本 tile z10/881/410)内的真实对象与属性分布, 作为下一步 `cm95_to_navsea_v1.yaml` 的唯一 MySQL 输入依据。 默认 AOI: z10 / x=881 / y=410 lon: 129.7265625 .. 130.078125 lat: 33.43144133557529 .. 33.7243396617476 该 z10 tile 已覆盖本次提供的: z11/1762/820 z12/3525/1643 本脚本: - 只读 MySQL; - 不修改 importer/schema; - 不生成 PBF; - 不依赖历史 V 系列结果; - 不需要 NEWPEC/旧 PBF 数据库; - 不涉及 style 修改。 输出: - AOI报告_v1.json - 对象类别汇总_v1.csv - 对象类别尺度分布_v1.csv - 属性字段汇总_v1.csv - 属性值域汇总_v1.csv - 对象字典_v1.csv - 属性字典_v1.csv - AOI_Cell清单_v1.csv - AOI_Feature样本_v1.jsonl - AOI_关联对象样本_v1.jsonl - AOI_对象映射准备表_v1.csv - 摘要_v1.txt - ZIP 执行示例: python .\cm95_karatsu_aoi_mapping_audit_v1.py ` --host 127.0.0.1 ` --port 3306 ` --user root ` --database cm93_chart ` --dataset-id 1 如不传密码,会安全提示输入。 """ from __future__ import annotations import argparse import csv import getpass import json import math import sys import zipfile from collections import Counter, defaultdict from datetime import datetime from pathlib import Path from typing import Any, Iterable try: import pymysql from pymysql.cursors import DictCursor except ImportError: print("缺少 pymysql,请执行: pip install pymysql") sys.exit(2) VERSION = "v1.0" def parse_args(): p = argparse.ArgumentParser( description="CM95 唐津 AOI → NavSea PBF 映射输入审计 v1" ) p.add_argument("--host", default="127.0.0.1") p.add_argument("--port", type=int, default=3306) p.add_argument("--user", default="root") p.add_argument("--password", default=None) p.add_argument("--database", default="cm93_chart") p.add_argument("--dataset-id", type=int, default=1) p.add_argument("--z", type=int, default=10) p.add_argument("--x", type=int, default=881) p.add_argument("--y", type=int, default=410) p.add_argument( "--sample-per-class", type=int, default=12, help="每个 object_class + geometry_type 最多保存多少条 Feature 样本", ) p.add_argument( "--value-sample-limit", type=int, default=40, help="每个属性值域最多在 CSV 中列出多少个高频值", ) p.add_argument("--output-dir", default=None) return p.parse_args() def tile_bounds(z: int, x: int, y: int): n = 2 ** z lon_min = x / n * 360.0 - 180.0 lon_max = (x + 1) / n * 360.0 - 180.0 def tile_lat(ty: int) -> float: return math.degrees( math.atan( math.sinh( math.pi * (1.0 - 2.0 * ty / n) ) ) ) lat_max = tile_lat(y) lat_min = tile_lat(y + 1) return lon_min, lat_min, lon_max, lat_max def begin_read_only(cur, warnings: list[str]): try: cur.execute("SET SESSION TRANSACTION READ ONLY") except Exception as e: warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") try: cur.execute("START TRANSACTION READ ONLY") except Exception as e: warnings.append(f"START TRANSACTION READ ONLY failed: {e}") cur.execute("START TRANSACTION") def scalar(cur, sql: str, params=()): cur.execute(sql, params) row = cur.fetchone() if not row: return None return next(iter(row.values())) def fetchall(cur, sql: str, params=()): cur.execute(sql, params) return list(cur.fetchall()) def write_csv(path: Path, rows: Iterable[dict[str, Any]]): rows = list(rows) if not rows: path.write_text("", encoding="utf-8-sig") return keys = [] seen = set() for row in rows: for key in row: if key not in seen: seen.add(key) keys.append(key) with path.open("w", encoding="utf-8-sig", newline="") as f: w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore") w.writeheader() for row in rows: out = {} for key in keys: value = row.get(key) if isinstance(value, (dict, list, tuple)): value = json.dumps( value, ensure_ascii=False, default=str, ) out[key] = value w.writerow(out) def json_obj(v): if v is None: return {} if isinstance(v, dict): return v if isinstance(v, (bytes, bytearray, memoryview)): v = bytes(v).decode("utf-8") obj = json.loads(v) return obj if isinstance(obj, dict) else {} def normalized_attr_value(row: dict[str, Any]): vt = row.get("value_type") if vt in ("BYTE", "INT", "LONG"): return row.get("value_int") if vt in ("REAL", "WORD10"): return row.get("value_real") if vt == "STRING": return row.get("value_text") if vt in ("LIST", "COMPLEX"): value = row.get("value_json") if value is not None: try: if isinstance(value, (dict, list)): return value return json.loads(value) except Exception: return str(value) return row.get("value_text") return ( row.get("value_text") if row.get("value_text") is not None else row.get("value_int") if row.get("value_int") is not None else row.get("value_real") ) def stable_value_text(value: Any) -> str: if isinstance(value, (dict, list)): return json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str, ) if value is None: return "" if isinstance(value, float): return format(value, ".15g") return str(value) def suggested_source_id( dataset_id: int, cell_path: str, source_record_index: int, geometry_type: str, ): # 这里只输出建议的稳定 identity 文本,不在本阶段分配 int32 fid。 return ( f"cm95:{dataset_id}:" f"{cell_path.replace(chr(92), '/')}:" f"{source_record_index}:" f"{geometry_type.lower()}" ) def main(): a = parse_args() if a.password is None: a.password = getpass.getpass("MySQL password: ") lon_min, lat_min, lon_max, lat_max = tile_bounds(a.z, a.x, a.y) stamp = datetime.now().strftime("%Y%m%d_%H%M%S") outdir = Path( a.output_dir or f"cm95_karatsu_mapping_audit_dataset_{a.dataset_id}_{stamp}" ).resolve() outdir.mkdir(parents=True, exist_ok=True) warnings: list[str] = [] conn = pymysql.connect( host=a.host, port=a.port, user=a.user, password=a.password, database=a.database, charset="utf8mb4", cursorclass=DictCursor, autocommit=False, connect_timeout=20, read_timeout=900, write_timeout=30, ) report: dict[str, Any] = { "version": VERSION, "generated_at": datetime.now().isoformat(timespec="seconds"), "database": a.database, "dataset_id": a.dataset_id, "read_only": True, "aoi": { "tile": { "z": a.z, "x": a.x, "y": a.y, }, "bbox_wgs84": { "min_lon": lon_min, "min_lat": lat_min, "max_lon": lon_max, "max_lat": lat_max, }, "known_navsea_samples_inside": [ "z10/881/410", "z11/1762/820", "z12/3525/1643", ], }, } bbox_where = """ f.dataset_id=%s AND f.bbox_min_lon IS NOT NULL AND f.bbox_min_lat IS NOT NULL AND f.bbox_max_lon IS NOT NULL AND f.bbox_max_lat IS NOT NULL AND f.bbox_max_lon >= %s AND f.bbox_min_lon <= %s AND f.bbox_max_lat >= %s AND f.bbox_min_lat <= %s """ bbox_params = ( a.dataset_id, lon_min, lon_max, lat_min, lat_max, ) try: with conn.cursor() as cur: begin_read_only(cur, warnings) print("[1/8] Dataset / AOI Cell ...") dataset = fetchall( cur, """ SELECT id, source_type, name, source_root, parser_version, normalized_srid FROM chart_dataset WHERE id=%s """, (a.dataset_id,), ) if not dataset: raise RuntimeError( f"Dataset ID={a.dataset_id} 不存在" ) report["dataset"] = dataset[0] cells = fetchall( cur, """ SELECT id AS cell_id, source_relative_path, source_file_name, scale_code, native_scale, cell_name, feature_count, attribute_count, bbox_min_lon, bbox_min_lat, bbox_max_lon, bbox_max_lat FROM chart_cell WHERE dataset_id=%s AND bbox_max_lon >= %s AND bbox_min_lon <= %s AND bbox_max_lat >= %s AND bbox_min_lat <= %s ORDER BY native_scale,source_relative_path """, ( a.dataset_id, lon_min, lon_max, lat_min, lat_max, ), ) write_csv(outdir / "AOI_Cell清单_v1.csv", cells) print("[2/8] AOI Feature 对象类别统计 ...") class_summary = fetchall( cur, f""" SELECT f.object_class_id, f.object_class, f.geometry_type, COUNT(*) AS feature_count, COUNT(DISTINCT f.cell_id) AS cell_count, MIN(f.native_scale) AS min_native_scale, MAX(f.native_scale) AS max_native_scale, SUM(CASE WHEN f.geometry_repaired=1 THEN 1 ELSE 0 END) AS repaired_count, SUM(CASE WHEN f.geometry_issue IS NOT NULL THEN 1 ELSE 0 END) AS issue_count FROM chart_feature f WHERE {bbox_where} GROUP BY f.object_class_id, f.object_class, f.geometry_type ORDER BY feature_count DESC, f.object_class, f.geometry_type """, bbox_params, ) object_dict = { int(r["object_class_id"]): r for r in fetchall( cur, """ SELECT object_class_id, object_class, geometry_code, display_code, description, raw_line FROM chart_dictionary_object WHERE dataset_id=%s """, (a.dataset_id,), ) } for row in class_summary: oid = row.get("object_class_id") d = object_dict.get(int(oid)) if oid is not None else None row["dictionary_geometry_code"] = ( d.get("geometry_code") if d else None ) row["dictionary_display_code"] = ( d.get("display_code") if d else None ) row["description"] = ( d.get("description") if d else None ) write_csv( outdir / "对象类别汇总_v1.csv", class_summary, ) scale_summary = fetchall( cur, f""" SELECT f.object_class_id, f.object_class, f.geometry_type, f.scale_code, f.native_scale, COUNT(*) AS feature_count FROM chart_feature f WHERE {bbox_where} GROUP BY f.object_class_id, f.object_class, f.geometry_type, f.scale_code, f.native_scale ORDER BY f.object_class, f.geometry_type, f.native_scale """, bbox_params, ) write_csv( outdir / "对象类别尺度分布_v1.csv", scale_summary, ) print("[3/8] 属性字段 / 值域统计 ...") attr_rows = fetchall( cur, f""" SELECT f.object_class_id, f.object_class, f.geometry_type, f.native_scale, a.attribute_id, a.attribute_name, a.value_type, a.value_int, a.value_real, a.value_text, a.value_json FROM chart_feature_attribute a JOIN chart_feature f ON f.id=a.feature_id WHERE {bbox_where} ORDER BY f.object_class, f.geometry_type, a.attribute_name """, bbox_params, ) attr_stats: dict[ tuple[Any, ...], dict[str, Any], ] = {} attr_value_counts: dict[ tuple[Any, ...], Counter[str], ] = defaultdict(Counter) attr_scale_values: dict[ tuple[Any, ...], set[int], ] = defaultdict(set) for row in attr_rows: key = ( row["object_class_id"], row["object_class"], row["geometry_type"], row["attribute_id"], row["attribute_name"], row["value_type"], ) if key not in attr_stats: attr_stats[key] = { "object_class_id": row["object_class_id"], "object_class": row["object_class"], "geometry_type": row["geometry_type"], "attribute_id": row["attribute_id"], "attribute_name": row["attribute_name"], "value_type": row["value_type"], "occurrence_count": 0, } attr_stats[key]["occurrence_count"] += 1 attr_scale_values[key].add(int(row["native_scale"])) value = normalized_attr_value(row) attr_value_counts[key][stable_value_text(value)] += 1 attr_summary_rows = [] attr_value_rows = [] for key, info in attr_stats.items(): values = attr_value_counts[key] info = dict(info) info["distinct_value_count"] = len(values) info["native_scales"] = sorted(attr_scale_values[key]) info["top_values"] = [ { "value": value, "count": count, } for value, count in values.most_common(a.value_sample_limit) ] attr_summary_rows.append(info) for rank, (value, count) in enumerate( values.most_common(a.value_sample_limit), start=1, ): attr_value_rows.append({ "object_class_id": key[0], "object_class": key[1], "geometry_type": key[2], "attribute_id": key[3], "attribute_name": key[4], "value_type": key[5], "rank": rank, "value": value, "count": count, "total_distinct_values": len(values), }) attr_summary_rows.sort( key=lambda r: ( str(r["object_class"]), str(r["geometry_type"]), str(r["attribute_name"]), ) ) write_csv( outdir / "属性字段汇总_v1.csv", attr_summary_rows, ) write_csv( outdir / "属性值域汇总_v1.csv", attr_value_rows, ) print("[4/8] Feature 样本 / attributes_json ...") sample_rows = fetchall( cur, f""" SELECT f.id AS feature_id, f.cell_id, c.source_relative_path, f.source_record_index, f.object_class_id, f.object_class, f.geometry_type, f.scale_code, f.native_scale, f.raw_geotype, f.coverage_id, f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, f.attributes_json, f.geometry_repaired, f.geometry_issue FROM chart_feature f JOIN chart_cell c ON c.id=f.cell_id WHERE {bbox_where} ORDER BY f.object_class, f.geometry_type, f.native_scale, c.source_relative_path, f.source_record_index """, bbox_params, ) class_seen: Counter[tuple[str, str]] = Counter() selected_samples: list[dict[str, Any]] = [] selected_feature_ids: set[int] = set() for row in sample_rows: key = ( str(row["object_class"]), str(row["geometry_type"]), ) if class_seen[key] >= a.sample_per_class: continue class_seen[key] += 1 attrs = json_obj(row.pop("attributes_json")) row["attributes"] = attrs row["suggested_source_id"] = suggested_source_id( a.dataset_id, str(row["source_relative_path"]), int(row["source_record_index"]), str(row["geometry_type"]), ) selected_samples.append(row) selected_feature_ids.add(int(row["feature_id"])) with ( outdir / "AOI_Feature样本_v1.jsonl" ).open("w", encoding="utf-8") as f: for row in selected_samples: f.write( json.dumps( row, ensure_ascii=False, default=str, ) + "\n" ) print("[5/8] 关联对象样本 ...") # 从样本里的 _cm93_related_indices 追踪同 Cell Feature, # 仅作为 mapping 调查辅助;不假设关系方向。 related_keys: set[tuple[int, int]] = set() for row in selected_samples: attrs = row.get("attributes") or {} related = attrs.get("_cm93_related_indices") if not isinstance(related, list): continue for value in related: try: related_keys.add( ( int(row["cell_id"]), int(value), ) ) except Exception: continue related_rows: list[dict[str, Any]] = [] # AOI 样本量有限,按 key 查询,避免构造超长 SQL IN。 for cell_id, source_index in sorted(related_keys): rows = fetchall( cur, """ SELECT f.id AS feature_id, f.cell_id, c.source_relative_path, f.source_record_index, f.object_class_id, f.object_class, f.geometry_type, f.scale_code, f.native_scale, f.raw_geotype, f.attributes_json FROM chart_feature f JOIN chart_cell c ON c.id=f.cell_id WHERE f.dataset_id=%s AND f.cell_id=%s AND f.source_record_index=%s LIMIT 1 """, ( a.dataset_id, cell_id, source_index, ), ) if not rows: continue row = rows[0] row["attributes"] = json_obj( row.pop("attributes_json") ) row["suggested_source_id"] = suggested_source_id( a.dataset_id, str(row["source_relative_path"]), int(row["source_record_index"]), str(row["geometry_type"]), ) related_rows.append(row) with ( outdir / "AOI_关联对象样本_v1.jsonl" ).open("w", encoding="utf-8") as f: for row in related_rows: f.write( json.dumps( row, ensure_ascii=False, default=str, ) + "\n" ) print("[6/8] Dictionary 子集 ...") seen_object_ids = { int(r["object_class_id"]) for r in class_summary if r.get("object_class_id") is not None } object_dict_rows = [ object_dict[oid] for oid in sorted(seen_object_ids) if oid in object_dict ] write_csv( outdir / "对象字典_v1.csv", object_dict_rows, ) seen_attr_ids = { int(r["attribute_id"]) for r in attr_summary_rows if r.get("attribute_id") is not None } all_attr_dict = fetchall( cur, """ SELECT attribute_id, attribute_name, source_type_name, decoder_type, min_value, max_value, raw_line FROM chart_dictionary_attribute WHERE dataset_id=%s ORDER BY attribute_id """, (a.dataset_id,), ) attr_dict_rows = [ r for r in all_attr_dict if int(r["attribute_id"]) in seen_attr_ids ] write_csv( outdir / "属性字典_v1.csv", attr_dict_rows, ) print("[7/8] NavSea mapping 准备表 ...") attrs_by_class: dict[ tuple[Any, Any, Any], list[str], ] = defaultdict(list) for row in attr_summary_rows: ckey = ( row["object_class_id"], row["object_class"], row["geometry_type"], ) attrs_by_class[ckey].append( str(row["attribute_name"]) ) mapping_prepare = [] for row in class_summary: key = ( row["object_class_id"], row["object_class"], row["geometry_type"], ) mapping_prepare.append({ "cm95_object_class_id": row["object_class_id"], "cm95_object_class": row["object_class"], "cm95_description": row.get("description"), "geometry_type": row["geometry_type"], "aoi_feature_count": row["feature_count"], "aoi_cell_count": row["cell_count"], "min_native_scale": row["min_native_scale"], "max_native_scale": row["max_native_scale"], "available_attributes": sorted( set(attrs_by_class.get(key, [])) ), "target_source_layer": "", "canonical_family": "", "canonical_object_type": "", "chart_render_type": "", "safety_class": "", "mapping_status": "UNMAPPED", "notes": "", }) write_csv( outdir / "AOI_对象映射准备表_v1.csv", mapping_prepare, ) print("[8/8] 汇总 ...") spatial_feature_count = int( scalar( cur, f""" SELECT COUNT(*) FROM chart_feature f WHERE {bbox_where} """, bbox_params, ) or 0 ) geometry_counts = fetchall( cur, f""" SELECT f.geometry_type, COUNT(*) AS feature_count FROM chart_feature f WHERE {bbox_where} GROUP BY f.geometry_type ORDER BY f.geometry_type """, bbox_params, ) report.update({ "warnings": warnings, "counts": { "aoi_cells": len(cells), "aoi_spatial_features": spatial_feature_count, "object_class_geometry_groups": len(class_summary), "attribute_group_count": len(attr_summary_rows), "sample_feature_count": len(selected_samples), "related_sample_count": len(related_rows), }, "geometry_counts": geometry_counts, "object_classes": [ { "object_class_id": r["object_class_id"], "object_class": r["object_class"], "geometry_type": r["geometry_type"], "feature_count": r["feature_count"], "description": r.get("description"), } for r in class_summary ], "next_step": ( "Use AOI_对象映射准备表_v1.csv + 属性值域汇总_v1.csv " "to author cm95_to_navsea_v1.yaml." ), }) conn.rollback() finally: conn.close() ( outdir / "AOI报告_v1.json" ).write_text( json.dumps( report, ensure_ascii=False, indent=2, default=str, ), encoding="utf-8", ) lines = [ "CM95 唐津 AOI → NavSea PBF 映射输入审计 v1", "=" * 84, f"Dataset ID : {a.dataset_id}", f"Tile : z{a.z}/{a.x}/{a.y}", ( "BBox WGS84 : " f"{lon_min:.10f},{lat_min:.10f} " f"→ {lon_max:.10f},{lat_max:.10f}" ), "", f"AOI cells : {report['counts']['aoi_cells']}", f"AOI spatial features : {report['counts']['aoi_spatial_features']}", ( "Object class/geom groups : " f"{report['counts']['object_class_geometry_groups']}" ), ( "Attribute groups : " f"{report['counts']['attribute_group_count']}" ), f"Feature samples : {report['counts']['sample_feature_count']}", f"Related samples : {report['counts']['related_sample_count']}", "", "Geometry counts:", ] for row in report["geometry_counts"]: lines.append( f" {str(row['geometry_type']):10s} " f"{int(row['feature_count'])}" ) lines += [ "", "NEXT:", " 上传本脚本生成的 ZIP。", " 下一步直接生成 cm95_to_navsea_v1.yaml 初版并标注 UNMAPPED / hard_gate 缺口。", ] summary = "\n".join(lines) ( outdir / "摘要_v1.txt" ).write_text(summary, encoding="utf-8") zip_path = outdir.with_suffix(".zip") with zipfile.ZipFile( zip_path, "w", compression=zipfile.ZIP_DEFLATED, ) as zf: for p in sorted(outdir.rglob("*")): if p.is_file(): zf.write( p, p.relative_to(outdir.parent), ) print("") print(summary) print("") print(f"报告目录: {outdir}") print(f"报告 ZIP : {zip_path}") print("把 ZIP 上传给 ChatGPT。") if __name__ == "__main__": main()