717 lines
21 KiB
Python
717 lines
21 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
r"""
|
||
CM95 / CM93 助航名称多尺度与显式关系审计 v2
|
||
|
||
目标
|
||
====
|
||
确认 CM95 自身是否存在可安全写入 NavSea
|
||
`navigation_marks.chart_label_text` 的名称来源。
|
||
|
||
重点：
|
||
1. 同时检查 1:20,000 / 1:50,000 / 1:200,000；
|
||
2. 优先检查助航对象和 `_texto` 中的显式关系/索引字段；
|
||
3. 再检查同 Cell 最近文字；
|
||
4. 不自动把最近文字当名称；
|
||
5. 不读取或反灌 NEWPEC/reference PBF 名称；
|
||
6. 只读 MySQL，不修改 importer/schema。
|
||
|
||
默认 AOI：
|
||
z10/914/385
|
||
lon 141.328125 .. 141.6796875
|
||
lat 40.44694705960049 .. 40.71395582628604
|
||
|
||
输出：
|
||
- cm95_nav_name_multiscale_audit_v2.json
|
||
- cm95_nav_name_multiscale_groups_v2.csv
|
||
- cm95_nav_text_relation_fields_v2.csv
|
||
- cm95_nav_name_multiscale_audit_v2.txt
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import getpass
|
||
import json
|
||
import math
|
||
from collections import Counter, defaultdict
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pymysql
|
||
from pymysql.cursors import DictCursor
|
||
|
||
|
||
NAV_CLASSES = {
|
||
"LIGHTS",
|
||
"BCNLAT",
|
||
"BCNSPP",
|
||
"BOYSPP",
|
||
"RTPBCN",
|
||
"TOWERS",
|
||
"_extgn",
|
||
"SISTAT",
|
||
"SISTAW",
|
||
}
|
||
|
||
DEFAULT_SCALES = (20000, 50000, 200000)
|
||
DIRECT_NAME_KEYS = ("OBJNAM", "NOBJNM")
|
||
|
||
RELATION_KEY_HINTS = (
|
||
"relation",
|
||
"related",
|
||
"index",
|
||
"record",
|
||
"assoc",
|
||
"link",
|
||
"ref",
|
||
"cm93",
|
||
)
|
||
|
||
DEFAULT_BBOX = (
|
||
141.328125,
|
||
40.44694705960049,
|
||
141.6796875,
|
||
40.71395582628604,
|
||
)
|
||
|
||
|
||
def parse_args():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--host", default="127.0.0.1")
|
||
p.add_argument("--port", type=int, default=3306)
|
||
p.add_argument("--user", default="root")
|
||
p.add_argument("--password", default=None)
|
||
p.add_argument("--database", default="cm93_chart")
|
||
p.add_argument("--dataset-id", type=int, default=1)
|
||
p.add_argument(
|
||
"--scales",
|
||
default="20000,50000,200000",
|
||
help="逗号分隔，例如 20000,50000,200000",
|
||
)
|
||
p.add_argument("--min-lon", type=float, default=DEFAULT_BBOX[0])
|
||
p.add_argument("--min-lat", type=float, default=DEFAULT_BBOX[1])
|
||
p.add_argument("--max-lon", type=float, default=DEFAULT_BBOX[2])
|
||
p.add_argument("--max-lat", type=float, default=DEFAULT_BBOX[3])
|
||
p.add_argument("--radius-m", type=float, default=800.0)
|
||
p.add_argument("--candidate-limit", type=int, default=12)
|
||
p.add_argument("--output-dir", default=".")
|
||
return p.parse_args()
|
||
|
||
|
||
def json_dict(value: Any) -> dict[str, Any]:
|
||
if value is None:
|
||
return {}
|
||
if isinstance(value, dict):
|
||
return value
|
||
if isinstance(value, (bytes, bytearray, memoryview)):
|
||
value = bytes(value).decode("utf-8", "replace")
|
||
try:
|
||
obj = json.loads(value)
|
||
except Exception:
|
||
return {}
|
||
return obj if isinstance(obj, dict) else {}
|
||
|
||
|
||
def relation20(attrs: dict[str, Any]):
|
||
value = attrs.get("_cm93_relation20_raw")
|
||
if value is None:
|
||
return None
|
||
try:
|
||
return int(value)
|
||
except Exception:
|
||
return str(value)
|
||
|
||
|
||
def texta(attrs: dict[str, Any]):
|
||
value = attrs.get("_texta")
|
||
if not isinstance(value, dict):
|
||
return None, None
|
||
text = value.get("text")
|
||
prefix = value.get("prefix")
|
||
if text is None:
|
||
return None, prefix
|
||
text = str(text).strip()
|
||
return (text or None), prefix
|
||
|
||
|
||
def direct_names(attrs: dict[str, Any]):
|
||
out = []
|
||
for key in DIRECT_NAME_KEYS:
|
||
value = attrs.get(key)
|
||
if isinstance(value, str) and value.strip():
|
||
out.append({"field": key, "text": value.strip()})
|
||
return out
|
||
|
||
|
||
def relation_like_fields(attrs: dict[str, Any]):
|
||
out = {}
|
||
for key, value in attrs.items():
|
||
lk = str(key).lower()
|
||
if any(h in lk for h in RELATION_KEY_HINTS):
|
||
out[key] = value
|
||
return out
|
||
|
||
|
||
def haversine_m(lon1, lat1, lon2, lat2):
|
||
r = 6371008.8
|
||
p1 = math.radians(lat1)
|
||
p2 = math.radians(lat2)
|
||
dp = p2 - p1
|
||
dl = math.radians(lon2 - lon1)
|
||
a = (
|
||
math.sin(dp / 2) ** 2
|
||
+ math.cos(p1) * math.cos(p2)
|
||
* math.sin(dl / 2) ** 2
|
||
)
|
||
return 2 * r * math.asin(min(1.0, math.sqrt(a)))
|
||
|
||
|
||
def flatten_relation_values(obj: Any):
|
||
"""用于审计，不解释语义。把标量/list/dict转成可比较字符串集合。"""
|
||
values = set()
|
||
|
||
def walk(v):
|
||
if v is None:
|
||
return
|
||
if isinstance(v, dict):
|
||
for kk, vv in v.items():
|
||
values.add(f"{kk}={json.dumps(vv, ensure_ascii=False, sort_keys=True, default=str)}")
|
||
walk(vv)
|
||
elif isinstance(v, list):
|
||
for item in v:
|
||
walk(item)
|
||
else:
|
||
values.add(str(v))
|
||
|
||
walk(obj)
|
||
return values
|
||
|
||
|
||
def write_csv(path: Path, rows, fields):
|
||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||
w = csv.DictWriter(f, fieldnames=fields)
|
||
w.writeheader()
|
||
for row in rows:
|
||
out = {}
|
||
for field in fields:
|
||
value = row.get(field)
|
||
if isinstance(value, (dict, list, tuple)):
|
||
value = json.dumps(
|
||
value,
|
||
ensure_ascii=False,
|
||
separators=(",", ":"),
|
||
default=str,
|
||
)
|
||
out[field] = value
|
||
w.writerow(out)
|
||
|
||
|
||
def main():
|
||
a = parse_args()
|
||
scales = tuple(
|
||
sorted(
|
||
{
|
||
int(x.strip())
|
||
for x in a.scales.split(",")
|
||
if x.strip()
|
||
}
|
||
)
|
||
)
|
||
if not scales:
|
||
raise SystemExit("scales 不能为空")
|
||
|
||
if a.password is None:
|
||
a.password = getpass.getpass("MySQL password: ")
|
||
|
||
outdir = Path(a.output_dir).resolve()
|
||
outdir.mkdir(parents=True, exist_ok=True)
|
||
|
||
conn = pymysql.connect(
|
||
host=a.host,
|
||
port=a.port,
|
||
user=a.user,
|
||
password=a.password,
|
||
database=a.database,
|
||
charset="utf8mb4",
|
||
cursorclass=DictCursor,
|
||
autocommit=False,
|
||
connect_timeout=20,
|
||
read_timeout=900,
|
||
)
|
||
|
||
nav_rows = []
|
||
text_rows = []
|
||
|
||
try:
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute("SET SESSION TRANSACTION READ ONLY")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
cur.execute("START TRANSACTION READ ONLY")
|
||
except Exception:
|
||
cur.execute("START TRANSACTION")
|
||
|
||
nav_ph = ",".join(["%s"] * len(NAV_CLASSES))
|
||
scale_ph = ",".join(["%s"] * len(scales))
|
||
|
||
cur.execute(
|
||
f"""
|
||
SELECT
|
||
f.id AS feature_id,
|
||
f.cell_id,
|
||
c.source_relative_path,
|
||
f.source_record_index,
|
||
f.object_class,
|
||
f.native_scale,
|
||
f.bbox_min_lon AS lon,
|
||
f.bbox_min_lat AS lat,
|
||
f.attributes_json
|
||
FROM chart_feature f
|
||
JOIN chart_cell c ON c.id=f.cell_id
|
||
WHERE f.dataset_id=%s
|
||
AND f.native_scale IN ({scale_ph})
|
||
AND f.geometry_type='POINT'
|
||
AND f.object_class IN ({nav_ph})
|
||
AND f.bbox_min_lon BETWEEN %s AND %s
|
||
AND f.bbox_min_lat BETWEEN %s AND %s
|
||
ORDER BY
|
||
f.native_scale,
|
||
f.cell_id,
|
||
f.source_record_index
|
||
""",
|
||
(
|
||
a.dataset_id,
|
||
*scales,
|
||
*sorted(NAV_CLASSES),
|
||
a.min_lon,
|
||
a.max_lon,
|
||
a.min_lat,
|
||
a.max_lat,
|
||
),
|
||
)
|
||
nav_rows = list(cur.fetchall())
|
||
|
||
lat_pad = a.radius_m / 111320.0
|
||
mid_lat = (a.min_lat + a.max_lat) / 2.0
|
||
lon_pad = (
|
||
a.radius_m
|
||
/ max(
|
||
1.0,
|
||
111320.0 * math.cos(math.radians(mid_lat)),
|
||
)
|
||
)
|
||
|
||
cur.execute(
|
||
f"""
|
||
SELECT
|
||
f.id AS feature_id,
|
||
f.cell_id,
|
||
c.source_relative_path,
|
||
f.source_record_index,
|
||
f.object_class,
|
||
f.native_scale,
|
||
f.bbox_min_lon AS lon,
|
||
f.bbox_min_lat AS lat,
|
||
f.attributes_json
|
||
FROM chart_feature f
|
||
JOIN chart_cell c ON c.id=f.cell_id
|
||
WHERE f.dataset_id=%s
|
||
AND f.native_scale IN ({scale_ph})
|
||
AND f.geometry_type='POINT'
|
||
AND f.object_class='_texto'
|
||
AND f.bbox_min_lon BETWEEN %s AND %s
|
||
AND f.bbox_min_lat BETWEEN %s AND %s
|
||
ORDER BY
|
||
f.native_scale,
|
||
f.cell_id,
|
||
f.source_record_index
|
||
""",
|
||
(
|
||
a.dataset_id,
|
||
*scales,
|
||
a.min_lon - lon_pad,
|
||
a.max_lon + lon_pad,
|
||
a.min_lat - lat_pad,
|
||
a.max_lat + lat_pad,
|
||
),
|
||
)
|
||
text_rows = list(cur.fetchall())
|
||
|
||
conn.rollback()
|
||
finally:
|
||
conn.close()
|
||
|
||
parsed_nav = []
|
||
for row in nav_rows:
|
||
attrs = json_dict(row.pop("attributes_json"))
|
||
parsed_nav.append(
|
||
{
|
||
**row,
|
||
"feature_id": int(row["feature_id"]),
|
||
"cell_id": int(row["cell_id"]),
|
||
"source_record_index": int(row["source_record_index"]),
|
||
"native_scale": int(row["native_scale"]),
|
||
"lon": float(row["lon"]),
|
||
"lat": float(row["lat"]),
|
||
"attributes": attrs,
|
||
"relation20": relation20(attrs),
|
||
"direct_names": direct_names(attrs),
|
||
"relation_fields": relation_like_fields(attrs),
|
||
}
|
||
)
|
||
|
||
parsed_text = []
|
||
relation_field_counter = Counter()
|
||
|
||
for row in text_rows:
|
||
attrs = json_dict(row.pop("attributes_json"))
|
||
text, prefix = texta(attrs)
|
||
rel_fields = relation_like_fields(attrs)
|
||
for key in rel_fields:
|
||
relation_field_counter[
|
||
(int(row["native_scale"]), str(key))
|
||
] += 1
|
||
|
||
parsed_text.append(
|
||
{
|
||
**row,
|
||
"feature_id": int(row["feature_id"]),
|
||
"cell_id": int(row["cell_id"]),
|
||
"source_record_index": int(row["source_record_index"]),
|
||
"native_scale": int(row["native_scale"]),
|
||
"lon": float(row["lon"]),
|
||
"lat": float(row["lat"]),
|
||
"text": text,
|
||
"prefix": prefix,
|
||
"attributes": attrs,
|
||
"relation_fields": rel_fields,
|
||
"relation_values": sorted(
|
||
flatten_relation_values(rel_fields)
|
||
),
|
||
}
|
||
)
|
||
|
||
# relation20 仅在同 cell + 同 scale 内组成助航主组。
|
||
grouped = defaultdict(list)
|
||
standalone = []
|
||
|
||
for row in parsed_nav:
|
||
rel = row["relation20"]
|
||
if rel is None:
|
||
standalone.append([row])
|
||
else:
|
||
grouped[
|
||
(
|
||
row["native_scale"],
|
||
row["cell_id"],
|
||
rel,
|
||
)
|
||
].append(row)
|
||
|
||
nav_groups = list(grouped.values()) + standalone
|
||
|
||
results = []
|
||
per_scale = {}
|
||
|
||
for members in nav_groups:
|
||
scale = members[0]["native_scale"]
|
||
cell_id = members[0]["cell_id"]
|
||
rel = members[0]["relation20"]
|
||
lon = sum(m["lon"] for m in members) / len(members)
|
||
lat = sum(m["lat"] for m in members) / len(members)
|
||
|
||
direct = []
|
||
nav_relation_values = set()
|
||
for m in members:
|
||
for n in m["direct_names"]:
|
||
direct.append(
|
||
{
|
||
"object_class": m["object_class"],
|
||
"source_record_index": m["source_record_index"],
|
||
**n,
|
||
}
|
||
)
|
||
nav_relation_values |= flatten_relation_values(
|
||
m["relation_fields"]
|
||
)
|
||
|
||
candidates = []
|
||
|
||
# 同比例尺；显式关系优先，同 cell 次之，距离只作证据。
|
||
for t in parsed_text:
|
||
if t["native_scale"] != scale or not t["text"]:
|
||
continue
|
||
|
||
distance = haversine_m(
|
||
lon,
|
||
lat,
|
||
t["lon"],
|
||
t["lat"],
|
||
)
|
||
|
||
explicit_overlap = sorted(
|
||
nav_relation_values.intersection(
|
||
set(t["relation_values"])
|
||
)
|
||
)
|
||
|
||
if distance <= a.radius_m or explicit_overlap:
|
||
candidates.append(
|
||
{
|
||
"feature_id": t["feature_id"],
|
||
"cell_id": t["cell_id"],
|
||
"source_relative_path": t["source_relative_path"],
|
||
"source_record_index": t["source_record_index"],
|
||
"native_scale": t["native_scale"],
|
||
"lon": t["lon"],
|
||
"lat": t["lat"],
|
||
"text": t["text"],
|
||
"prefix": t["prefix"],
|
||
"same_cell": t["cell_id"] == cell_id,
|
||
"distance_m": round(distance, 2),
|
||
"explicit_relation_overlap": explicit_overlap,
|
||
"text_relation_fields": t["relation_fields"],
|
||
}
|
||
)
|
||
|
||
candidates.sort(
|
||
key=lambda c: (
|
||
not bool(c["explicit_relation_overlap"]),
|
||
not c["same_cell"],
|
||
c["distance_m"],
|
||
c["text"],
|
||
)
|
||
)
|
||
|
||
explicit = [
|
||
c
|
||
for c in candidates
|
||
if c["explicit_relation_overlap"]
|
||
]
|
||
same = [c for c in candidates if c["same_cell"]]
|
||
|
||
nearest_same = (
|
||
min(same, key=lambda c: c["distance_m"])
|
||
if same
|
||
else None
|
||
)
|
||
|
||
result = {
|
||
"native_scale": scale,
|
||
"cell_id": cell_id,
|
||
"source_relative_path": members[0]["source_relative_path"],
|
||
"relation20": rel,
|
||
"lon": lon,
|
||
"lat": lat,
|
||
"component_classes": [
|
||
m["object_class"] for m in members
|
||
],
|
||
"component_source_record_indices": [
|
||
m["source_record_index"] for m in members
|
||
],
|
||
"direct_names": direct,
|
||
"nav_relation_fields": {
|
||
str(m["source_record_index"]): m["relation_fields"]
|
||
for m in members
|
||
if m["relation_fields"]
|
||
},
|
||
"explicit_text_candidates": explicit[
|
||
: a.candidate_limit
|
||
],
|
||
"nearest_same_cell_text": (
|
||
nearest_same["text"] if nearest_same else None
|
||
),
|
||
"nearest_same_cell_distance_m": (
|
||
nearest_same["distance_m"] if nearest_same else None
|
||
),
|
||
"same_cell_candidate_count": len(same),
|
||
"all_candidate_count": len(candidates),
|
||
"candidate_summary": candidates[
|
||
: a.candidate_limit
|
||
],
|
||
}
|
||
results.append(result)
|
||
|
||
results.sort(
|
||
key=lambda r: (
|
||
r["native_scale"],
|
||
r["cell_id"],
|
||
-1 if r["relation20"] is None else r["relation20"],
|
||
r["lon"],
|
||
r["lat"],
|
||
)
|
||
)
|
||
|
||
for scale in scales:
|
||
subset = [r for r in results if r["native_scale"] == scale]
|
||
per_scale[str(scale)] = {
|
||
"nav_groups": len(subset),
|
||
"groups_with_direct_names": sum(
|
||
1 for r in subset if r["direct_names"]
|
||
),
|
||
"groups_with_explicit_text_candidates": sum(
|
||
1
|
||
for r in subset
|
||
if r["explicit_text_candidates"]
|
||
),
|
||
"groups_with_same_cell_text_within_50m": sum(
|
||
1
|
||
for r in subset
|
||
if r["nearest_same_cell_distance_m"] is not None
|
||
and r["nearest_same_cell_distance_m"] <= 50
|
||
),
|
||
"groups_with_same_cell_text_within_100m": sum(
|
||
1
|
||
for r in subset
|
||
if r["nearest_same_cell_distance_m"] is not None
|
||
and r["nearest_same_cell_distance_m"] <= 100
|
||
),
|
||
"groups_with_same_cell_text_within_250m": sum(
|
||
1
|
||
for r in subset
|
||
if r["nearest_same_cell_distance_m"] is not None
|
||
and r["nearest_same_cell_distance_m"] <= 250
|
||
),
|
||
}
|
||
|
||
relation_rows = [
|
||
{
|
||
"native_scale": scale,
|
||
"field_name": field,
|
||
"occurrence_count": count,
|
||
}
|
||
for (scale, field), count in sorted(
|
||
relation_field_counter.items()
|
||
)
|
||
]
|
||
|
||
report = {
|
||
"schema": "navsea.cm95.nav-name-multiscale-audit/v2",
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"dataset_id": a.dataset_id,
|
||
"scales": list(scales),
|
||
"aoi_bbox": [
|
||
a.min_lon,
|
||
a.min_lat,
|
||
a.max_lon,
|
||
a.max_lat,
|
||
],
|
||
"search_radius_m": a.radius_m,
|
||
"counts": {
|
||
"nav_component_rows": len(parsed_nav),
|
||
"nav_groups": len(results),
|
||
"texto_rows": len(parsed_text),
|
||
"texto_rows_with_text": sum(
|
||
1 for t in parsed_text if t["text"]
|
||
),
|
||
},
|
||
"per_scale": per_scale,
|
||
"text_relation_field_counts": relation_rows,
|
||
"groups": results,
|
||
"decision": "READY_FOR_SAFE_NAME_RULE_REVIEW",
|
||
}
|
||
|
||
json_path = (
|
||
outdir / "cm95_nav_name_multiscale_audit_v2.json"
|
||
)
|
||
groups_csv = (
|
||
outdir / "cm95_nav_name_multiscale_groups_v2.csv"
|
||
)
|
||
rel_csv = (
|
||
outdir / "cm95_nav_text_relation_fields_v2.csv"
|
||
)
|
||
txt_path = (
|
||
outdir / "cm95_nav_name_multiscale_audit_v2.txt"
|
||
)
|
||
|
||
json_path.write_text(
|
||
json.dumps(
|
||
report,
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
default=str,
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
write_csv(
|
||
groups_csv,
|
||
results,
|
||
[
|
||
"native_scale",
|
||
"cell_id",
|
||
"source_relative_path",
|
||
"relation20",
|
||
"lon",
|
||
"lat",
|
||
"component_classes",
|
||
"component_source_record_indices",
|
||
"direct_names",
|
||
"nav_relation_fields",
|
||
"explicit_text_candidates",
|
||
"nearest_same_cell_text",
|
||
"nearest_same_cell_distance_m",
|
||
"same_cell_candidate_count",
|
||
"all_candidate_count",
|
||
"candidate_summary",
|
||
],
|
||
)
|
||
|
||
write_csv(
|
||
rel_csv,
|
||
relation_rows,
|
||
[
|
||
"native_scale",
|
||
"field_name",
|
||
"occurrence_count",
|
||
],
|
||
)
|
||
|
||
lines = [
|
||
"CM95 / CM93 助航名称多尺度与显式关系审计 v2",
|
||
"=" * 82,
|
||
f"Dataset ID : {a.dataset_id}",
|
||
f"Scales : {','.join(map(str, scales))}",
|
||
f"Nav component rows : {len(parsed_nav)}",
|
||
f"Nav groups : {len(results)}",
|
||
f"_texto rows : {len(parsed_text)}",
|
||
"",
|
||
]
|
||
|
||
for scale in scales:
|
||
s = per_scale[str(scale)]
|
||
lines += [
|
||
f"1:{scale}",
|
||
f" groups : {s['nav_groups']}",
|
||
f" direct names : {s['groups_with_direct_names']}",
|
||
f" explicit text relation: {s['groups_with_explicit_text_candidates']}",
|
||
f" same-cell <= 50m : {s['groups_with_same_cell_text_within_50m']}",
|
||
f" same-cell <= 100m : {s['groups_with_same_cell_text_within_100m']}",
|
||
f" same-cell <= 250m : {s['groups_with_same_cell_text_within_250m']}",
|
||
"",
|
||
]
|
||
|
||
lines += [
|
||
"注意：",
|
||
"- 显式关系候选优先于距离候选。",
|
||
"- 最近文字仍不会被脚本自动写入 chart_label_text。",
|
||
"- 若 20k/50k/200k 都无安全来源，则该数据源的助航名称保持为空，不从 NEWPEC 反灌。",
|
||
]
|
||
txt_path.write_text("\n".join(lines), encoding="utf-8")
|
||
|
||
print("\n".join(lines))
|
||
print("")
|
||
print(json_path)
|
||
print(groups_csv)
|
||
print(rel_csv)
|
||
print(txt_path)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|