#!/usr/bin/env python3 # -*- coding: utf-8 -*- r""" CM95 / CM93 助航名称多尺度与显式关系审计 v2 目标 ==== 确认 CM95 自身是否存在可安全写入 NavSea `navigation_marks.chart_label_text` 的名称来源。 重点： 1. 同时检查 1:20,000 / 1:50,000 / 1:200,000； 2. 优先检查助航对象和 `_texto` 中的显式关系/索引字段； 3. 再检查同 Cell 最近文字； 4. 不自动把最近文字当名称； 5. 不读取或反灌 NEWPEC/reference PBF 名称； 6. 只读 MySQL，不修改 importer/schema。 默认 AOI： z10/914/385 lon 141.328125 .. 141.6796875 lat 40.44694705960049 .. 40.71395582628604 输出： - cm95_nav_name_multiscale_audit_v2.json - cm95_nav_name_multiscale_groups_v2.csv - cm95_nav_text_relation_fields_v2.csv - cm95_nav_name_multiscale_audit_v2.txt """ from __future__ import annotations import argparse import csv import getpass import json import math from collections import Counter, defaultdict from datetime import datetime from pathlib import Path from typing import Any import pymysql from pymysql.cursors import DictCursor NAV_CLASSES = { "LIGHTS", "BCNLAT", "BCNSPP", "BOYSPP", "RTPBCN", "TOWERS", "_extgn", "SISTAT", "SISTAW", } DEFAULT_SCALES = (20000, 50000, 200000) DIRECT_NAME_KEYS = ("OBJNAM", "NOBJNM") RELATION_KEY_HINTS = ( "relation", "related", "index", "record", "assoc", "link", "ref", "cm93", ) DEFAULT_BBOX = ( 141.328125, 40.44694705960049, 141.6796875, 40.71395582628604, ) def parse_args(): p = argparse.ArgumentParser() p.add_argument("--host", default="127.0.0.1") p.add_argument("--port", type=int, default=3306) p.add_argument("--user", default="root") p.add_argument("--password", default=None) p.add_argument("--database", default="cm93_chart") p.add_argument("--dataset-id", type=int, default=1) p.add_argument( "--scales", default="20000,50000,200000", help="逗号分隔，例如 20000,50000,200000", ) p.add_argument("--min-lon", type=float, default=DEFAULT_BBOX[0]) p.add_argument("--min-lat", type=float, default=DEFAULT_BBOX[1]) p.add_argument("--max-lon", type=float, default=DEFAULT_BBOX[2]) p.add_argument("--max-lat", type=float, default=DEFAULT_BBOX[3]) p.add_argument("--radius-m", type=float, default=800.0) p.add_argument("--candidate-limit", type=int, default=12) p.add_argument("--output-dir", default=".") return p.parse_args() def json_dict(value: Any) -> dict[str, Any]: if value is None: return {} if isinstance(value, dict): return value if isinstance(value, (bytes, bytearray, memoryview)): value = bytes(value).decode("utf-8", "replace") try: obj = json.loads(value) except Exception: return {} return obj if isinstance(obj, dict) else {} def relation20(attrs: dict[str, Any]): value = attrs.get("_cm93_relation20_raw") if value is None: return None try: return int(value) except Exception: return str(value) def texta(attrs: dict[str, Any]): value = attrs.get("_texta") if not isinstance(value, dict): return None, None text = value.get("text") prefix = value.get("prefix") if text is None: return None, prefix text = str(text).strip() return (text or None), prefix def direct_names(attrs: dict[str, Any]): out = [] for key in DIRECT_NAME_KEYS: value = attrs.get(key) if isinstance(value, str) and value.strip(): out.append({"field": key, "text": value.strip()}) return out def relation_like_fields(attrs: dict[str, Any]): out = {} for key, value in attrs.items(): lk = str(key).lower() if any(h in lk for h in RELATION_KEY_HINTS): out[key] = value return out def haversine_m(lon1, lat1, lon2, lat2): r = 6371008.8 p1 = math.radians(lat1) p2 = math.radians(lat2) dp = p2 - p1 dl = math.radians(lon2 - lon1) a = ( math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2 ) return 2 * r * math.asin(min(1.0, math.sqrt(a))) def flatten_relation_values(obj: Any): """用于审计，不解释语义。把标量/list/dict转成可比较字符串集合。""" values = set() def walk(v): if v is None: return if isinstance(v, dict): for kk, vv in v.items(): values.add(f"{kk}={json.dumps(vv, ensure_ascii=False, sort_keys=True, default=str)}") walk(vv) elif isinstance(v, list): for item in v: walk(item) else: values.add(str(v)) walk(obj) return values def write_csv(path: Path, rows, fields): with path.open("w", encoding="utf-8-sig", newline="") as f: w = csv.DictWriter(f, fieldnames=fields) w.writeheader() for row in rows: out = {} for field in fields: value = row.get(field) if isinstance(value, (dict, list, tuple)): value = json.dumps( value, ensure_ascii=False, separators=(",", ":"), default=str, ) out[field] = value w.writerow(out) def main(): a = parse_args() scales = tuple( sorted( { int(x.strip()) for x in a.scales.split(",") if x.strip() } ) ) if not scales: raise SystemExit("scales 不能为空") if a.password is None: a.password = getpass.getpass("MySQL password: ") outdir = Path(a.output_dir).resolve() outdir.mkdir(parents=True, exist_ok=True) conn = pymysql.connect( host=a.host, port=a.port, user=a.user, password=a.password, database=a.database, charset="utf8mb4", cursorclass=DictCursor, autocommit=False, connect_timeout=20, read_timeout=900, ) nav_rows = [] text_rows = [] try: with conn.cursor() as cur: try: cur.execute("SET SESSION TRANSACTION READ ONLY") except Exception: pass try: cur.execute("START TRANSACTION READ ONLY") except Exception: cur.execute("START TRANSACTION") nav_ph = ",".join(["%s"] * len(NAV_CLASSES)) scale_ph = ",".join(["%s"] * len(scales)) cur.execute( f""" SELECT f.id AS feature_id, f.cell_id, c.source_relative_path, f.source_record_index, f.object_class, f.native_scale, f.bbox_min_lon AS lon, f.bbox_min_lat AS lat, f.attributes_json FROM chart_feature f JOIN chart_cell c ON c.id=f.cell_id WHERE f.dataset_id=%s AND f.native_scale IN ({scale_ph}) AND f.geometry_type='POINT' AND f.object_class IN ({nav_ph}) AND f.bbox_min_lon BETWEEN %s AND %s AND f.bbox_min_lat BETWEEN %s AND %s ORDER BY f.native_scale, f.cell_id, f.source_record_index """, ( a.dataset_id, *scales, *sorted(NAV_CLASSES), a.min_lon, a.max_lon, a.min_lat, a.max_lat, ), ) nav_rows = list(cur.fetchall()) lat_pad = a.radius_m / 111320.0 mid_lat = (a.min_lat + a.max_lat) / 2.0 lon_pad = ( a.radius_m / max( 1.0, 111320.0 * math.cos(math.radians(mid_lat)), ) ) cur.execute( f""" SELECT f.id AS feature_id, f.cell_id, c.source_relative_path, f.source_record_index, f.object_class, f.native_scale, f.bbox_min_lon AS lon, f.bbox_min_lat AS lat, f.attributes_json FROM chart_feature f JOIN chart_cell c ON c.id=f.cell_id WHERE f.dataset_id=%s AND f.native_scale IN ({scale_ph}) AND f.geometry_type='POINT' AND f.object_class='_texto' AND f.bbox_min_lon BETWEEN %s AND %s AND f.bbox_min_lat BETWEEN %s AND %s ORDER BY f.native_scale, f.cell_id, f.source_record_index """, ( a.dataset_id, *scales, a.min_lon - lon_pad, a.max_lon + lon_pad, a.min_lat - lat_pad, a.max_lat + lat_pad, ), ) text_rows = list(cur.fetchall()) conn.rollback() finally: conn.close() parsed_nav = [] for row in nav_rows: attrs = json_dict(row.pop("attributes_json")) parsed_nav.append( { **row, "feature_id": int(row["feature_id"]), "cell_id": int(row["cell_id"]), "source_record_index": int(row["source_record_index"]), "native_scale": int(row["native_scale"]), "lon": float(row["lon"]), "lat": float(row["lat"]), "attributes": attrs, "relation20": relation20(attrs), "direct_names": direct_names(attrs), "relation_fields": relation_like_fields(attrs), } ) parsed_text = [] relation_field_counter = Counter() for row in text_rows: attrs = json_dict(row.pop("attributes_json")) text, prefix = texta(attrs) rel_fields = relation_like_fields(attrs) for key in rel_fields: relation_field_counter[ (int(row["native_scale"]), str(key)) ] += 1 parsed_text.append( { **row, "feature_id": int(row["feature_id"]), "cell_id": int(row["cell_id"]), "source_record_index": int(row["source_record_index"]), "native_scale": int(row["native_scale"]), "lon": float(row["lon"]), "lat": float(row["lat"]), "text": text, "prefix": prefix, "attributes": attrs, "relation_fields": rel_fields, "relation_values": sorted( flatten_relation_values(rel_fields) ), } ) # relation20 仅在同 cell + 同 scale 内组成助航主组。 grouped = defaultdict(list) standalone = [] for row in parsed_nav: rel = row["relation20"] if rel is None: standalone.append([row]) else: grouped[ ( row["native_scale"], row["cell_id"], rel, ) ].append(row) nav_groups = list(grouped.values()) + standalone results = [] per_scale = {} for members in nav_groups: scale = members[0]["native_scale"] cell_id = members[0]["cell_id"] rel = members[0]["relation20"] lon = sum(m["lon"] for m in members) / len(members) lat = sum(m["lat"] for m in members) / len(members) direct = [] nav_relation_values = set() for m in members: for n in m["direct_names"]: direct.append( { "object_class": m["object_class"], "source_record_index": m["source_record_index"], **n, } ) nav_relation_values |= flatten_relation_values( m["relation_fields"] ) candidates = [] # 同比例尺；显式关系优先，同 cell 次之，距离只作证据。 for t in parsed_text: if t["native_scale"] != scale or not t["text"]: continue distance = haversine_m( lon, lat, t["lon"], t["lat"], ) explicit_overlap = sorted( nav_relation_values.intersection( set(t["relation_values"]) ) ) if distance <= a.radius_m or explicit_overlap: candidates.append( { "feature_id": t["feature_id"], "cell_id": t["cell_id"], "source_relative_path": t["source_relative_path"], "source_record_index": t["source_record_index"], "native_scale": t["native_scale"], "lon": t["lon"], "lat": t["lat"], "text": t["text"], "prefix": t["prefix"], "same_cell": t["cell_id"] == cell_id, "distance_m": round(distance, 2), "explicit_relation_overlap": explicit_overlap, "text_relation_fields": t["relation_fields"], } ) candidates.sort( key=lambda c: ( not bool(c["explicit_relation_overlap"]), not c["same_cell"], c["distance_m"], c["text"], ) ) explicit = [ c for c in candidates if c["explicit_relation_overlap"] ] same = [c for c in candidates if c["same_cell"]] nearest_same = ( min(same, key=lambda c: c["distance_m"]) if same else None ) result = { "native_scale": scale, "cell_id": cell_id, "source_relative_path": members[0]["source_relative_path"], "relation20": rel, "lon": lon, "lat": lat, "component_classes": [ m["object_class"] for m in members ], "component_source_record_indices": [ m["source_record_index"] for m in members ], "direct_names": direct, "nav_relation_fields": { str(m["source_record_index"]): m["relation_fields"] for m in members if m["relation_fields"] }, "explicit_text_candidates": explicit[ : a.candidate_limit ], "nearest_same_cell_text": ( nearest_same["text"] if nearest_same else None ), "nearest_same_cell_distance_m": ( nearest_same["distance_m"] if nearest_same else None ), "same_cell_candidate_count": len(same), "all_candidate_count": len(candidates), "candidate_summary": candidates[ : a.candidate_limit ], } results.append(result) results.sort( key=lambda r: ( r["native_scale"], r["cell_id"], -1 if r["relation20"] is None else r["relation20"], r["lon"], r["lat"], ) ) for scale in scales: subset = [r for r in results if r["native_scale"] == scale] per_scale[str(scale)] = { "nav_groups": len(subset), "groups_with_direct_names": sum( 1 for r in subset if r["direct_names"] ), "groups_with_explicit_text_candidates": sum( 1 for r in subset if r["explicit_text_candidates"] ), "groups_with_same_cell_text_within_50m": sum( 1 for r in subset if r["nearest_same_cell_distance_m"] is not None and r["nearest_same_cell_distance_m"] <= 50 ), "groups_with_same_cell_text_within_100m": sum( 1 for r in subset if r["nearest_same_cell_distance_m"] is not None and r["nearest_same_cell_distance_m"] <= 100 ), "groups_with_same_cell_text_within_250m": sum( 1 for r in subset if r["nearest_same_cell_distance_m"] is not None and r["nearest_same_cell_distance_m"] <= 250 ), } relation_rows = [ { "native_scale": scale, "field_name": field, "occurrence_count": count, } for (scale, field), count in sorted( relation_field_counter.items() ) ] report = { "schema": "navsea.cm95.nav-name-multiscale-audit/v2", "generated_at": datetime.now().isoformat(timespec="seconds"), "dataset_id": a.dataset_id, "scales": list(scales), "aoi_bbox": [ a.min_lon, a.min_lat, a.max_lon, a.max_lat, ], "search_radius_m": a.radius_m, "counts": { "nav_component_rows": len(parsed_nav), "nav_groups": len(results), "texto_rows": len(parsed_text), "texto_rows_with_text": sum( 1 for t in parsed_text if t["text"] ), }, "per_scale": per_scale, "text_relation_field_counts": relation_rows, "groups": results, "decision": "READY_FOR_SAFE_NAME_RULE_REVIEW", } json_path = ( outdir / "cm95_nav_name_multiscale_audit_v2.json" ) groups_csv = ( outdir / "cm95_nav_name_multiscale_groups_v2.csv" ) rel_csv = ( outdir / "cm95_nav_text_relation_fields_v2.csv" ) txt_path = ( outdir / "cm95_nav_name_multiscale_audit_v2.txt" ) json_path.write_text( json.dumps( report, ensure_ascii=False, indent=2, default=str, ), encoding="utf-8", ) write_csv( groups_csv, results, [ "native_scale", "cell_id", "source_relative_path", "relation20", "lon", "lat", "component_classes", "component_source_record_indices", "direct_names", "nav_relation_fields", "explicit_text_candidates", "nearest_same_cell_text", "nearest_same_cell_distance_m", "same_cell_candidate_count", "all_candidate_count", "candidate_summary", ], ) write_csv( rel_csv, relation_rows, [ "native_scale", "field_name", "occurrence_count", ], ) lines = [ "CM95 / CM93 助航名称多尺度与显式关系审计 v2", "=" * 82, f"Dataset ID : {a.dataset_id}", f"Scales : {','.join(map(str, scales))}", f"Nav component rows : {len(parsed_nav)}", f"Nav groups : {len(results)}", f"_texto rows : {len(parsed_text)}", "", ] for scale in scales: s = per_scale[str(scale)] lines += [ f"1:{scale}", f" groups : {s['nav_groups']}", f" direct names : {s['groups_with_direct_names']}", f" explicit text relation: {s['groups_with_explicit_text_candidates']}", f" same-cell <= 50m : {s['groups_with_same_cell_text_within_50m']}", f" same-cell <= 100m : {s['groups_with_same_cell_text_within_100m']}", f" same-cell <= 250m : {s['groups_with_same_cell_text_within_250m']}", "", ] lines += [ "注意：", "- 显式关系候选优先于距离候选。", "- 最近文字仍不会被脚本自动写入 chart_label_text。", "- 若 20k/50k/200k 都无安全来源，则该数据源的助航名称保持为空，不从 NEWPEC 反灌。", ] txt_path.write_text("\n".join(lines), encoding="utf-8") print("\n".join(lines)) print("") print(json_path) print(groups_csv) print(rel_csv) print(txt_path) if __name__ == "__main__": main()