CM93 MySQL importer validated baseline
This commit is contained in:
kaotei
977
cm93_mysql_invalid_area_audit_v4.py
Normal file
977
cm93_mysql_invalid_area_audit_v4.py
Normal file
@@ -0,0 +1,977 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
CM93 -> MySQL 无效 Area / Coverage 深度分类审计
|
||||
Version: v4.0
|
||||
|
||||
背景：
|
||||
- Dataset ID=1 为日本沿海局部 CM93 数据子集，不是完整 CM93。
|
||||
- V3 已确认：
|
||||
* Point 113346：全部有效
|
||||
* Line 121773：全部有效
|
||||
* Sounding 275700：全部有效
|
||||
* Area 81080：52 个 Shapely invalid
|
||||
* Coverage 1156：3 个 Shapely invalid
|
||||
* feature ↔ geometry table mapping issue = 0
|
||||
* raw_geotype=16 / META 的 4406 条为 EXPECTED_PARENT_NO_GEOMETRY
|
||||
|
||||
本脚本目标：
|
||||
1. 重新找出 invalid Area / Coverage；
|
||||
2. 对每一个 Shapely 报告的 invalidity point 做局部几何分析；
|
||||
3. 区分：
|
||||
- invalid 点本身位于 Cell 边界；
|
||||
- geometry 只是“整体碰 Cell 边界”，但 invalid 点在内部；
|
||||
- 重复顶点自接触；
|
||||
- 线段内部真正交叉；
|
||||
- shell / hole / ring 结构问题；
|
||||
- Interior disconnected；
|
||||
4. 找出多个 Feature 共享同一 invalid point 的簇；
|
||||
5. 找出 Area 与 Coverage 在同一 Cell、同一 invalid point 的关联；
|
||||
6. 输出当前数据库中 topology/edge/node/primitive 等相关 schema 线索，供下一阶段追 importer。
|
||||
7. 全程只读，不修 geometry，不改 MySQL，不做 PBF。
|
||||
|
||||
重要：
|
||||
“当前 Dataset 不是完整 CM93”只影响跨未导入海域的连续性判断。
|
||||
一个已经生成的 Polygon 如果在自身内部发生真正 segment crossing，
|
||||
仍属于需要调查的内在 geometry 问题。
|
||||
但如果 invalidity point 正好位于 Cell 裁切边界、并被多个对象/coverage 共用，
|
||||
更可能是 Cell clipping / shared topology / ring assembly 类问题，不能直接归为原始数据损坏。
|
||||
|
||||
依赖：
|
||||
pip install pymysql shapely
|
||||
|
||||
运行：
|
||||
python cm93_mysql_invalid_area_audit_v4.py ^
|
||||
--host 127.0.0.1 ^
|
||||
--port 3306 ^
|
||||
--user root ^
|
||||
--database cm93_chart ^
|
||||
--dataset-id 1
|
||||
|
||||
输出目录会自动压缩成 zip，直接把 zip 发给 ChatGPT。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import getpass
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
import zipfile
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||||
|
||||
try:
|
||||
import pymysql
|
||||
from pymysql.cursors import DictCursor
|
||||
except ImportError:
|
||||
print("缺少 pymysql：pip install pymysql shapely")
|
||||
sys.exit(2)
|
||||
|
||||
try:
|
||||
try:
|
||||
from shapely import from_wkb as _from_wkb
|
||||
def load_wkb(data: bytes):
|
||||
return _from_wkb(data)
|
||||
except ImportError:
|
||||
from shapely import wkb as _wkb
|
||||
def load_wkb(data: bytes):
|
||||
return _wkb.loads(data)
|
||||
|
||||
from shapely.geometry import Point, LineString
|
||||
from shapely.validation import explain_validity
|
||||
except ImportError:
|
||||
print("缺少 shapely：pip install pymysql shapely")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
INVALID_POINT_RE = re.compile(
|
||||
r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+"
|
||||
r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]"
|
||||
)
|
||||
|
||||
|
||||
def qi(name: str) -> str:
|
||||
return "`" + name.replace("`", "``") + "`"
|
||||
|
||||
|
||||
def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any:
|
||||
cur.execute(sql, params)
|
||||
row = cur.fetchone()
|
||||
return next(iter(row.values())) if row else None
|
||||
|
||||
|
||||
def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]:
|
||||
cur.execute(sql, params)
|
||||
return list(cur.fetchall())
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None):
|
||||
rows = list(rows)
|
||||
if fieldnames is None:
|
||||
seen = set()
|
||||
fieldnames = []
|
||||
for row in rows:
|
||||
for k in row.keys():
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
fieldnames.append(k)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
if not fieldnames:
|
||||
return
|
||||
w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
for row in rows:
|
||||
out = {}
|
||||
for k in fieldnames:
|
||||
v = row.get(k)
|
||||
if isinstance(v, (tuple, list, dict)):
|
||||
v = json.dumps(v, ensure_ascii=False)
|
||||
out[k] = "" if v is None else v
|
||||
w.writerow(out)
|
||||
|
||||
|
||||
def begin_read_only(cur, warnings: List[str]):
|
||||
try:
|
||||
cur.execute("SET SESSION TRANSACTION READ ONLY")
|
||||
except Exception as e:
|
||||
warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}")
|
||||
try:
|
||||
cur.execute("START TRANSACTION READ ONLY")
|
||||
except Exception as e:
|
||||
warnings.append(f"START TRANSACTION READ ONLY failed: {e}")
|
||||
try:
|
||||
cur.execute("START TRANSACTION")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def probe_wkb_fn(cur) -> str:
|
||||
for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"):
|
||||
try:
|
||||
cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1")
|
||||
cur.fetchone()
|
||||
return fn
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError("找不到可用的 geometry -> WKB 函数")
|
||||
|
||||
|
||||
def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]:
|
||||
if not reason:
|
||||
return None
|
||||
m = INVALID_POINT_RE.search(reason)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
return float(m.group(1)), float(m.group(2))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def bbox_from_row(row: Dict[str, Any], prefix: str = "") -> Optional[Tuple[float, float, float, float]]:
|
||||
keys = [
|
||||
prefix + "bbox_min_lon",
|
||||
prefix + "bbox_min_lat",
|
||||
prefix + "bbox_max_lon",
|
||||
prefix + "bbox_max_lat",
|
||||
]
|
||||
vals = []
|
||||
for k in keys:
|
||||
try:
|
||||
v = float(row[k])
|
||||
if not math.isfinite(v):
|
||||
return None
|
||||
vals.append(v)
|
||||
except Exception:
|
||||
return None
|
||||
return vals[0], vals[1], vals[2], vals[3]
|
||||
|
||||
|
||||
def point_cell_edge_context(
|
||||
p: Optional[Tuple[float, float]],
|
||||
bbox: Optional[Tuple[float, float, float, float]],
|
||||
tol: float,
|
||||
) -> Tuple[bool, str, Optional[float]]:
|
||||
if not p or not bbox:
|
||||
return False, "", None
|
||||
x, y = p
|
||||
minx, miny, maxx, maxy = bbox
|
||||
d = {
|
||||
"WEST": abs(x - minx),
|
||||
"EAST": abs(x - maxx),
|
||||
"SOUTH": abs(y - miny),
|
||||
"NORTH": abs(y - maxy),
|
||||
}
|
||||
edge, dist = min(d.items(), key=lambda kv: kv[1])
|
||||
return dist <= tol, edge if dist <= tol else "", dist
|
||||
|
||||
|
||||
def point_feature_bbox_edge_context(
|
||||
p: Optional[Tuple[float, float]],
|
||||
bbox: Optional[Tuple[float, float, float, float]],
|
||||
tol: float,
|
||||
) -> Tuple[bool, str, Optional[float]]:
|
||||
return point_cell_edge_context(p, bbox, tol)
|
||||
|
||||
|
||||
def ring_coordinate_lists(geom) -> List[Tuple[str, int, List[Tuple[float, float]]]]:
|
||||
out = []
|
||||
|
||||
def add_polygon(poly, poly_index: int):
|
||||
try:
|
||||
out.append((f"poly{poly_index}.shell", 0, [(float(x), float(y)) for x, y, *rest in poly.exterior.coords]))
|
||||
except Exception:
|
||||
pass
|
||||
for i, ring in enumerate(poly.interiors):
|
||||
try:
|
||||
out.append((f"poly{poly_index}.hole", i, [(float(x), float(y)) for x, y, *rest in ring.coords]))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if geom.geom_type == "Polygon":
|
||||
add_polygon(geom, 0)
|
||||
elif geom.geom_type == "MultiPolygon":
|
||||
for pi, poly in enumerate(geom.geoms):
|
||||
add_polygon(poly, pi)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def point_close(a: Tuple[float, float], b: Tuple[float, float], tol: float) -> bool:
|
||||
return abs(a[0]-b[0]) <= tol and abs(a[1]-b[1]) <= tol
|
||||
|
||||
|
||||
def analyze_invalid_point_on_rings(
|
||||
geom,
|
||||
invalid_point: Optional[Tuple[float, float]],
|
||||
tol: float,
|
||||
) -> Dict[str, Any]:
|
||||
result = {
|
||||
"polygon_count": 0,
|
||||
"ring_count": 0,
|
||||
"hole_count": 0,
|
||||
"total_vertices": 0,
|
||||
"invalid_point_vertex_occurrences": 0,
|
||||
"invalid_point_distinct_ring_hits": 0,
|
||||
"invalid_point_segment_hits": 0,
|
||||
"invalid_point_segment_interior_hits": 0,
|
||||
"invalid_point_endpoint_hits": 0,
|
||||
"repeated_nonconsecutive_vertex_count": 0,
|
||||
"zero_length_segment_count": 0,
|
||||
"mechanism": "UNRESOLVED",
|
||||
"ring_hit_labels": "",
|
||||
}
|
||||
|
||||
if geom.geom_type == "Polygon":
|
||||
result["polygon_count"] = 1
|
||||
elif geom.geom_type == "MultiPolygon":
|
||||
result["polygon_count"] = len(geom.geoms)
|
||||
else:
|
||||
return result
|
||||
|
||||
rings = ring_coordinate_lists(geom)
|
||||
result["ring_count"] = len(rings)
|
||||
result["hole_count"] = sum(1 for label, _, _ in rings if ".hole" in label)
|
||||
|
||||
ring_hits = set()
|
||||
vertex_occurrences = 0
|
||||
segment_hits = 0
|
||||
segment_interior_hits = 0
|
||||
endpoint_hits = 0
|
||||
repeated_nonconsecutive = 0
|
||||
zero_segments = 0
|
||||
|
||||
pgeom = Point(invalid_point) if invalid_point else None
|
||||
|
||||
for label, ring_index, coords in rings:
|
||||
result["total_vertices"] += len(coords)
|
||||
|
||||
# 去掉 ring 最后一个闭合重复点后做“非连续重复顶点”统计。
|
||||
core = coords[:-1] if len(coords) > 1 and point_close(coords[0], coords[-1], tol) else coords[:]
|
||||
seen_positions: Dict[Tuple[int, int], List[int]] = defaultdict(list)
|
||||
scale = 1.0 / max(tol, 1e-12)
|
||||
for i, c in enumerate(core):
|
||||
key = (round(c[0] * scale), round(c[1] * scale))
|
||||
seen_positions[key].append(i)
|
||||
for positions in seen_positions.values():
|
||||
if len(positions) > 1:
|
||||
# 相邻重复点不归到 nonconsecutive；它们通常会形成 zero segment。
|
||||
has_nonconsecutive = False
|
||||
for i in range(len(positions)):
|
||||
for j in range(i+1, len(positions)):
|
||||
if abs(positions[j] - positions[i]) > 1:
|
||||
has_nonconsecutive = True
|
||||
break
|
||||
if has_nonconsecutive:
|
||||
break
|
||||
if has_nonconsecutive:
|
||||
repeated_nonconsecutive += 1
|
||||
|
||||
if invalid_point:
|
||||
for c in core:
|
||||
if point_close(c, invalid_point, tol):
|
||||
vertex_occurrences += 1
|
||||
ring_hits.add(f"{label}[{ring_index}]")
|
||||
|
||||
for i in range(len(coords)-1):
|
||||
a = coords[i]
|
||||
b = coords[i+1]
|
||||
|
||||
if point_close(a, b, tol):
|
||||
zero_segments += 1
|
||||
|
||||
if not invalid_point:
|
||||
continue
|
||||
|
||||
try:
|
||||
seg = LineString([a, b])
|
||||
if seg.distance(pgeom) <= tol:
|
||||
segment_hits += 1
|
||||
ring_hits.add(f"{label}[{ring_index}]")
|
||||
at_a = point_close(a, invalid_point, tol)
|
||||
at_b = point_close(b, invalid_point, tol)
|
||||
if at_a or at_b:
|
||||
endpoint_hits += 1
|
||||
else:
|
||||
segment_interior_hits += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
result["invalid_point_vertex_occurrences"] = vertex_occurrences
|
||||
result["invalid_point_distinct_ring_hits"] = len(ring_hits)
|
||||
result["invalid_point_segment_hits"] = segment_hits
|
||||
result["invalid_point_segment_interior_hits"] = segment_interior_hits
|
||||
result["invalid_point_endpoint_hits"] = endpoint_hits
|
||||
result["repeated_nonconsecutive_vertex_count"] = repeated_nonconsecutive
|
||||
result["zero_length_segment_count"] = zero_segments
|
||||
result["ring_hit_labels"] = "|".join(sorted(ring_hits))
|
||||
|
||||
# 机制分类：只描述几何表现，不直接断言根因。
|
||||
if not invalid_point:
|
||||
result["mechanism"] = "NO_INVALID_POINT_FROM_VALIDATOR"
|
||||
elif segment_interior_hits >= 2 and vertex_occurrences == 0:
|
||||
result["mechanism"] = "TRUE_SEGMENT_CROSSING"
|
||||
elif vertex_occurrences >= 2:
|
||||
result["mechanism"] = "REPEATED_VERTEX_SELF_TOUCH"
|
||||
elif vertex_occurrences >= 1 and segment_interior_hits >= 1:
|
||||
result["mechanism"] = "VERTEX_TOUCHES_NONADJACENT_SEGMENT"
|
||||
elif len(ring_hits) >= 2:
|
||||
result["mechanism"] = "MULTI_RING_TOUCH_OR_CROSS"
|
||||
elif segment_hits >= 3:
|
||||
result["mechanism"] = "MULTI_SEGMENT_TOUCH"
|
||||
elif segment_hits >= 2:
|
||||
result["mechanism"] = "TWO_SEGMENT_INTERACTION"
|
||||
else:
|
||||
result["mechanism"] = "UNRESOLVED_LOCAL_TOPOLOGY"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def validator_reason_category(reason: str) -> str:
|
||||
s = (reason or "").lower()
|
||||
if "ring self-intersection" in s:
|
||||
return "RING_SELF_INTERSECTION"
|
||||
if "self-intersection" in s:
|
||||
return "SELF_INTERSECTION"
|
||||
if "interior is disconnected" in s:
|
||||
return "INTERIOR_DISCONNECTED"
|
||||
if "hole lies outside shell" in s:
|
||||
return "HOLE_OUTSIDE_SHELL"
|
||||
if "nested shells" in s:
|
||||
return "NESTED_SHELLS"
|
||||
if "nested holes" in s:
|
||||
return "NESTED_HOLES"
|
||||
if "too few points" in s:
|
||||
return "TOO_FEW_POINTS"
|
||||
return "OTHER_INVALID_GEOMETRY"
|
||||
|
||||
|
||||
def scan_invalid_geometries(
|
||||
cur,
|
||||
*,
|
||||
table: str,
|
||||
kind: str,
|
||||
dataset_id: int,
|
||||
wkb_fn: str,
|
||||
chunk_size: int,
|
||||
point_tol: float,
|
||||
edge_tol: float,
|
||||
) -> List[Dict[str, Any]]:
|
||||
last_id = 0
|
||||
invalids = []
|
||||
id_col = "feature_id" if kind == "AREA" else "id"
|
||||
|
||||
while True:
|
||||
if kind == "AREA":
|
||||
sql = f"""
|
||||
SELECT
|
||||
g.feature_id AS row_id,
|
||||
g.feature_id,
|
||||
NULL AS coverage_id,
|
||||
{wkb_fn}(g.geom) AS geom_wkb,
|
||||
f.object_class,
|
||||
f.raw_geotype,
|
||||
f.geometry_type,
|
||||
f.geometry_issue,
|
||||
f.geometry_repaired,
|
||||
f.source_record_index,
|
||||
f.cell_id,
|
||||
c.cell_name,
|
||||
f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat,
|
||||
c.bbox_min_lon AS cell_bbox_min_lon,
|
||||
c.bbox_min_lat AS cell_bbox_min_lat,
|
||||
c.bbox_max_lon AS cell_bbox_max_lon,
|
||||
c.bbox_max_lat AS cell_bbox_max_lat
|
||||
FROM chart_area_geometry g
|
||||
JOIN chart_feature f ON f.id=g.feature_id
|
||||
JOIN chart_cell c ON c.id=f.cell_id
|
||||
WHERE f.dataset_id=%s
|
||||
AND g.feature_id>%s
|
||||
ORDER BY g.feature_id
|
||||
LIMIT %s
|
||||
"""
|
||||
else:
|
||||
sql = f"""
|
||||
SELECT
|
||||
v.id AS row_id,
|
||||
NULL AS feature_id,
|
||||
v.id AS coverage_id,
|
||||
{wkb_fn}(v.geom) AS geom_wkb,
|
||||
'COVERAGE' AS object_class,
|
||||
NULL AS raw_geotype,
|
||||
'COVERAGE' AS geometry_type,
|
||||
v.geometry_issue,
|
||||
v.geometry_repaired,
|
||||
v.source_record_index,
|
||||
v.cell_id,
|
||||
c.cell_name,
|
||||
NULL AS bbox_min_lon, NULL AS bbox_min_lat,
|
||||
NULL AS bbox_max_lon, NULL AS bbox_max_lat,
|
||||
c.bbox_min_lon AS cell_bbox_min_lon,
|
||||
c.bbox_min_lat AS cell_bbox_min_lat,
|
||||
c.bbox_max_lon AS cell_bbox_max_lon,
|
||||
c.bbox_max_lat AS cell_bbox_max_lat
|
||||
FROM chart_coverage v
|
||||
JOIN chart_cell c ON c.id=v.cell_id
|
||||
WHERE v.dataset_id=%s
|
||||
AND v.id>%s
|
||||
ORDER BY v.id
|
||||
LIMIT %s
|
||||
"""
|
||||
|
||||
cur.execute(sql, (dataset_id, last_id, chunk_size))
|
||||
batch = list(cur.fetchall())
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for r in batch:
|
||||
last_id = int(r["row_id"])
|
||||
wkb = r["geom_wkb"]
|
||||
if isinstance(wkb, memoryview):
|
||||
wkb = wkb.tobytes()
|
||||
elif isinstance(wkb, bytearray):
|
||||
wkb = bytes(wkb)
|
||||
|
||||
try:
|
||||
geom = load_wkb(wkb)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
try:
|
||||
valid = bool(geom.is_valid)
|
||||
except Exception:
|
||||
continue
|
||||
if valid:
|
||||
continue
|
||||
|
||||
reason = explain_validity(geom)
|
||||
p = parse_invalid_point(reason)
|
||||
cb = bbox_from_row(r, "cell_")
|
||||
fb = bbox_from_row(r)
|
||||
|
||||
on_cell_edge, cell_edge, cell_edge_dist = point_cell_edge_context(p, cb, edge_tol)
|
||||
on_feature_bbox_edge, feature_bbox_edge, feature_bbox_edge_dist = point_feature_bbox_edge_context(
|
||||
p, fb, edge_tol
|
||||
)
|
||||
|
||||
local = analyze_invalid_point_on_rings(geom, p, point_tol)
|
||||
|
||||
geom_bounds = tuple(float(v) for v in geom.bounds)
|
||||
|
||||
row = {
|
||||
"kind": kind,
|
||||
"feature_id": r.get("feature_id"),
|
||||
"coverage_id": r.get("coverage_id"),
|
||||
"object_class": r.get("object_class"),
|
||||
"cell_id": r.get("cell_id"),
|
||||
"cell_name": r.get("cell_name"),
|
||||
"source_record_index": r.get("source_record_index"),
|
||||
"raw_geotype": r.get("raw_geotype"),
|
||||
"geometry_type": r.get("geometry_type"),
|
||||
"db_geometry_issue": r.get("geometry_issue"),
|
||||
"db_geometry_repaired": r.get("geometry_repaired"),
|
||||
"validator_category": validator_reason_category(reason),
|
||||
"validator_reason": reason,
|
||||
"invalid_x": p[0] if p else None,
|
||||
"invalid_y": p[1] if p else None,
|
||||
"cell_bbox": cb,
|
||||
"feature_bbox": fb,
|
||||
"geometry_bounds": geom_bounds,
|
||||
"invalid_point_on_cell_edge": int(on_cell_edge),
|
||||
"cell_edge": cell_edge,
|
||||
"cell_edge_distance_deg": cell_edge_dist,
|
||||
"invalid_point_on_feature_bbox_edge": int(on_feature_bbox_edge),
|
||||
"feature_bbox_edge": feature_bbox_edge,
|
||||
"feature_bbox_edge_distance_deg": feature_bbox_edge_dist,
|
||||
}
|
||||
row.update(local)
|
||||
invalids.append(row)
|
||||
|
||||
print(f" {kind:<8} scanned_to={last_id} invalid_found={len(invalids)}")
|
||||
|
||||
return invalids
|
||||
|
||||
|
||||
def rounded_point_key(row: Dict[str, Any], decimals: int = 7) -> Optional[Tuple[float, float]]:
|
||||
try:
|
||||
x = float(row["invalid_x"])
|
||||
y = float(row["invalid_y"])
|
||||
if math.isfinite(x) and math.isfinite(y):
|
||||
return round(x, decimals), round(y, decimals)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def build_shared_point_groups(all_rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
groups = defaultdict(list)
|
||||
for r in all_rows:
|
||||
k = rounded_point_key(r)
|
||||
if k is not None:
|
||||
groups[k].append(r)
|
||||
|
||||
out = []
|
||||
for p, rs in groups.items():
|
||||
if len(rs) < 2:
|
||||
continue
|
||||
kinds = Counter(r["kind"] for r in rs)
|
||||
cells = sorted({str(r.get("cell_name") or "") for r in rs})
|
||||
objs = Counter(str(r.get("object_class") or "") for r in rs)
|
||||
out.append({
|
||||
"invalid_x": p[0],
|
||||
"invalid_y": p[1],
|
||||
"record_count": len(rs),
|
||||
"area_count": kinds.get("AREA", 0),
|
||||
"coverage_count": kinds.get("COVERAGE", 0),
|
||||
"cells": "|".join(cells),
|
||||
"object_classes": "|".join(f"{k}:{v}" for k, v in objs.most_common()),
|
||||
"cell_edge_records": sum(int(r.get("invalid_point_on_cell_edge") or 0) for r in rs),
|
||||
"feature_ids": "|".join(str(r["feature_id"]) for r in rs if r.get("feature_id") is not None),
|
||||
"coverage_ids": "|".join(str(r["coverage_id"]) for r in rs if r.get("coverage_id") is not None),
|
||||
})
|
||||
|
||||
out.sort(key=lambda r: (-r["record_count"], r["invalid_x"], r["invalid_y"]))
|
||||
return out
|
||||
|
||||
|
||||
def mark_area_coverage_links(
|
||||
areas: List[Dict[str, Any]],
|
||||
coverages: List[Dict[str, Any]],
|
||||
decimals: int = 7,
|
||||
):
|
||||
cov_index = defaultdict(list)
|
||||
for c in coverages:
|
||||
p = rounded_point_key(c, decimals)
|
||||
if p is not None:
|
||||
cov_index[(str(c.get("cell_name")), p)].append(c)
|
||||
|
||||
for a in areas:
|
||||
p = rounded_point_key(a, decimals)
|
||||
matches = cov_index.get((str(a.get("cell_name")), p), []) if p is not None else []
|
||||
a["matches_invalid_coverage_same_cell_point"] = int(bool(matches))
|
||||
a["matching_coverage_ids"] = "|".join(
|
||||
str(x.get("coverage_id")) for x in matches
|
||||
)
|
||||
|
||||
|
||||
def schema_topology_inventory(cur, database: str) -> List[Dict[str, Any]]:
|
||||
keywords = (
|
||||
"edge", "node", "vector", "primitive", "segment",
|
||||
"geom", "geometry", "coverage", "ring", "topolog"
|
||||
)
|
||||
rs = fetch_all(cur, """
|
||||
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA=%s
|
||||
ORDER BY TABLE_NAME, ORDINAL_POSITION
|
||||
""", (database,))
|
||||
out = []
|
||||
for r in rs:
|
||||
blob = (r["TABLE_NAME"] + " " + r["COLUMN_NAME"]).lower()
|
||||
if any(k in blob for k in keywords):
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def table_inventory(cur, database: str) -> List[Dict[str, Any]]:
|
||||
return fetch_all(cur, """
|
||||
SELECT TABLE_NAME, TABLE_ROWS, ENGINE, TABLE_COMMENT
|
||||
FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA=%s
|
||||
ORDER BY TABLE_NAME
|
||||
""", (database,))
|
||||
|
||||
|
||||
def classify_for_next_step(r: Dict[str, Any]) -> str:
|
||||
"""
|
||||
这是“调查优先级/表现分类”，不是最终根因。
|
||||
"""
|
||||
vc = r.get("validator_category")
|
||||
mechanism = r.get("mechanism")
|
||||
on_edge = bool(r.get("invalid_point_on_cell_edge"))
|
||||
coverage_link = bool(r.get("matches_invalid_coverage_same_cell_point"))
|
||||
|
||||
if vc == "INTERIOR_DISCONNECTED":
|
||||
return "INTERNAL_RING_STRUCTURE_NEEDS_REVIEW"
|
||||
|
||||
if on_edge and coverage_link:
|
||||
return "CELL_BOUNDARY_SHARED_WITH_COVERAGE"
|
||||
if on_edge and mechanism in {
|
||||
"REPEATED_VERTEX_SELF_TOUCH",
|
||||
"VERTEX_TOUCHES_NONADJACENT_SEGMENT",
|
||||
"MULTI_RING_TOUCH_OR_CROSS",
|
||||
"MULTI_SEGMENT_TOUCH",
|
||||
"TWO_SEGMENT_INTERACTION",
|
||||
}:
|
||||
return "CELL_BOUNDARY_TOPOLOGY_CANDIDATE"
|
||||
if on_edge and mechanism == "TRUE_SEGMENT_CROSSING":
|
||||
return "CELL_BOUNDARY_SEGMENT_CROSSING"
|
||||
if not on_edge and mechanism == "TRUE_SEGMENT_CROSSING":
|
||||
return "INTERNAL_TRUE_SEGMENT_CROSSING"
|
||||
if not on_edge and mechanism == "REPEATED_VERTEX_SELF_TOUCH":
|
||||
return "INTERNAL_REPEATED_VERTEX_SELF_TOUCH"
|
||||
if not on_edge:
|
||||
return "INTERNAL_INVALID_GEOMETRY_NEEDS_REVIEW"
|
||||
return "BOUNDARY_INVALID_GEOMETRY_NEEDS_REVIEW"
|
||||
|
||||
|
||||
def zip_dir(folder: Path) -> Path:
|
||||
zpath = folder.with_suffix(".zip")
|
||||
with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z:
|
||||
for p in sorted(folder.rglob("*")):
|
||||
if p.is_file():
|
||||
z.write(p, p.relative_to(folder.parent))
|
||||
return zpath
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser(description="CM93 invalid Area/Coverage deep audit v4")
|
||||
p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1"))
|
||||
p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306")))
|
||||
p.add_argument("--user", default=os.getenv("MYSQL_USER", "root"))
|
||||
p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD"))
|
||||
p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart"))
|
||||
p.add_argument("--dataset-id", type=int, default=1)
|
||||
p.add_argument("--chunk-size", type=int, default=2000)
|
||||
p.add_argument(
|
||||
"--point-tolerance",
|
||||
type=float,
|
||||
default=1e-8,
|
||||
help="invalid point 与 ring 顶点/segment 的比较容差（degree）"
|
||||
)
|
||||
p.add_argument(
|
||||
"--cell-edge-tolerance",
|
||||
type=float,
|
||||
default=5e-6,
|
||||
help="invalidity point 是否位于 Cell bbox 边界的容差（degree）"
|
||||
)
|
||||
p.add_argument("--output", default=None)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if not args.password:
|
||||
args.password = getpass.getpass("MySQL password: ")
|
||||
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
outdir = Path(args.output or f"cm93_invalid_area_audit_v4_dataset_{args.dataset_id}_{stamp}")
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
report = {
|
||||
"version": "v4.0",
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"database": args.database,
|
||||
"dataset_id": args.dataset_id,
|
||||
"read_only": True,
|
||||
"pbf": "OUT_OF_SCOPE",
|
||||
"dataset_scope_note": (
|
||||
"CM93JAPAN 只是局部子集。跨未导入海域连续性不作为 FAIL；"
|
||||
"本轮只深挖已经生成的 invalid Polygon/Coverage 自身。"
|
||||
),
|
||||
"warnings": [],
|
||||
}
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = pymysql.connect(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
user=args.user,
|
||||
password=args.password,
|
||||
database=args.database,
|
||||
charset="utf8mb4",
|
||||
cursorclass=DictCursor,
|
||||
autocommit=False,
|
||||
connect_timeout=20,
|
||||
read_timeout=600,
|
||||
write_timeout=30,
|
||||
)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
begin_read_only(cur, report["warnings"])
|
||||
report["mysql_version"] = scalar(cur, "SELECT VERSION()")
|
||||
wkb_fn = probe_wkb_fn(cur)
|
||||
report["wkb_fn"] = wkb_fn
|
||||
|
||||
print("[1/6] 深扫 invalid Area ...")
|
||||
areas = scan_invalid_geometries(
|
||||
cur,
|
||||
table="chart_area_geometry",
|
||||
kind="AREA",
|
||||
dataset_id=args.dataset_id,
|
||||
wkb_fn=wkb_fn,
|
||||
chunk_size=args.chunk_size,
|
||||
point_tol=args.point_tolerance,
|
||||
edge_tol=args.cell_edge_tolerance,
|
||||
)
|
||||
|
||||
print("[2/6] 深扫 invalid Coverage ...")
|
||||
coverages = scan_invalid_geometries(
|
||||
cur,
|
||||
table="chart_coverage",
|
||||
kind="COVERAGE",
|
||||
dataset_id=args.dataset_id,
|
||||
wkb_fn=wkb_fn,
|
||||
chunk_size=args.chunk_size,
|
||||
point_tol=args.point_tolerance,
|
||||
edge_tol=args.cell_edge_tolerance,
|
||||
)
|
||||
|
||||
print("[3/6] 建立 Area ↔ Coverage 同点关联 ...")
|
||||
mark_area_coverage_links(areas, coverages)
|
||||
|
||||
for r in areas:
|
||||
r["next_step_class"] = classify_for_next_step(r)
|
||||
for r in coverages:
|
||||
r["matches_invalid_coverage_same_cell_point"] = 0
|
||||
r["matching_coverage_ids"] = ""
|
||||
r["next_step_class"] = classify_for_next_step(r)
|
||||
|
||||
all_invalid = areas + coverages
|
||||
shared_groups = build_shared_point_groups(all_invalid)
|
||||
|
||||
print("[4/6] 导出 schema/topology 线索 ...")
|
||||
topology_schema = schema_topology_inventory(cur, args.database)
|
||||
tables = table_inventory(cur, args.database)
|
||||
|
||||
print("[5/6] 汇总分类 ...")
|
||||
validator_counts = Counter(r["validator_category"] for r in all_invalid)
|
||||
mechanism_counts = Counter(r["mechanism"] for r in all_invalid)
|
||||
next_counts = Counter(r["next_step_class"] for r in all_invalid)
|
||||
area_next_counts = Counter(r["next_step_class"] for r in areas)
|
||||
cov_next_counts = Counter(r["next_step_class"] for r in coverages)
|
||||
|
||||
summary = {
|
||||
"invalid_area_count": len(areas),
|
||||
"invalid_coverage_count": len(coverages),
|
||||
"invalid_total": len(all_invalid),
|
||||
"invalid_point_on_cell_edge_total": sum(int(r["invalid_point_on_cell_edge"]) for r in all_invalid),
|
||||
"invalid_point_internal_total": sum(1 for r in all_invalid if not r["invalid_point_on_cell_edge"]),
|
||||
"area_invalid_point_on_cell_edge": sum(int(r["invalid_point_on_cell_edge"]) for r in areas),
|
||||
"area_invalid_point_internal": sum(1 for r in areas if not r["invalid_point_on_cell_edge"]),
|
||||
"coverage_invalid_point_on_cell_edge": sum(int(r["invalid_point_on_cell_edge"]) for r in coverages),
|
||||
"area_matches_invalid_coverage_same_cell_point": sum(
|
||||
int(r.get("matches_invalid_coverage_same_cell_point") or 0)
|
||||
for r in areas
|
||||
),
|
||||
"validator_categories": dict(validator_counts),
|
||||
"mechanisms": dict(mechanism_counts),
|
||||
"next_step_classes": dict(next_counts),
|
||||
"area_next_step_classes": dict(area_next_counts),
|
||||
"coverage_next_step_classes": dict(cov_next_counts),
|
||||
"shared_invalid_point_group_count": len(shared_groups),
|
||||
}
|
||||
report["summary"] = summary
|
||||
|
||||
write_csv(outdir / "invalid_areas_deep_classified.csv", areas)
|
||||
write_csv(outdir / "invalid_coverages_deep_classified.csv", coverages)
|
||||
write_csv(outdir / "invalid_all_deep_classified.csv", all_invalid)
|
||||
write_csv(outdir / "shared_invalid_point_groups.csv", shared_groups)
|
||||
write_csv(outdir / "topology_schema_inventory.csv", topology_schema)
|
||||
write_csv(outdir / "database_table_inventory.csv", tables)
|
||||
|
||||
write_csv(
|
||||
outdir / "validator_category_counts.csv",
|
||||
[{"category": k, "count": v} for k, v in validator_counts.most_common()],
|
||||
["category", "count"]
|
||||
)
|
||||
write_csv(
|
||||
outdir / "local_mechanism_counts.csv",
|
||||
[{"mechanism": k, "count": v} for k, v in mechanism_counts.most_common()],
|
||||
["mechanism", "count"]
|
||||
)
|
||||
write_csv(
|
||||
outdir / "next_step_class_counts.csv",
|
||||
[{"next_step_class": k, "count": v} for k, v in next_counts.most_common()],
|
||||
["next_step_class", "count"]
|
||||
)
|
||||
|
||||
# 重点样本：每一 next_step_class 最多 10 条
|
||||
samples = []
|
||||
sc = Counter()
|
||||
for r in sorted(
|
||||
all_invalid,
|
||||
key=lambda x: (
|
||||
x.get("next_step_class", ""),
|
||||
str(x.get("cell_name", "")),
|
||||
int(x.get("feature_id") or x.get("coverage_id") or 0)
|
||||
)
|
||||
):
|
||||
c = r["next_step_class"]
|
||||
if sc[c] < 10:
|
||||
samples.append(r)
|
||||
sc[c] += 1
|
||||
write_csv(outdir / "deep_classification_samples.csv", samples)
|
||||
|
||||
(outdir / "invalid_area_audit_report_v4.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, default=str),
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
lines = []
|
||||
lines.append("CM93 -> MySQL Invalid Area/Coverage 深度分类审计 v4")
|
||||
lines.append("=" * 82)
|
||||
lines.append(f"Generated: {report['generated_at']}")
|
||||
lines.append(f"Database: {args.database}")
|
||||
lines.append(f"Dataset ID: {args.dataset_id}")
|
||||
lines.append(f"MySQL: {report['mysql_version']}")
|
||||
lines.append("")
|
||||
lines.append("DATASET SCOPE")
|
||||
lines.append(" 当前 CM93JAPAN 是局部数据，不要求与未导入海域连续。")
|
||||
lines.append(" 但本轮检查的是已生成 Polygon 本身的自交/自接触机制。")
|
||||
lines.append("")
|
||||
lines.append("COUNTS")
|
||||
lines.append(f" invalid Area: {len(areas)}")
|
||||
lines.append(f" invalid Coverage: {len(coverages)}")
|
||||
lines.append(f" total: {len(all_invalid)}")
|
||||
lines.append("")
|
||||
lines.append("INVALID POINT LOCATION")
|
||||
lines.append(
|
||||
f" Area invalid point ON cell edge: "
|
||||
f"{summary['area_invalid_point_on_cell_edge']}"
|
||||
)
|
||||
lines.append(
|
||||
f" Area invalid point INTERNAL: "
|
||||
f"{summary['area_invalid_point_internal']}"
|
||||
)
|
||||
lines.append(
|
||||
f" Coverage invalid point ON cell edge:"
|
||||
f" {summary['coverage_invalid_point_on_cell_edge']}"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("AREA ↔ COVERAGE")
|
||||
lines.append(
|
||||
f" Area same cell + same invalid point as invalid Coverage: "
|
||||
f"{summary['area_matches_invalid_coverage_same_cell_point']}"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("VALIDATOR CATEGORY")
|
||||
for k, v in validator_counts.most_common():
|
||||
lines.append(f" {k:42s} {v}")
|
||||
lines.append("")
|
||||
lines.append("LOCAL MECHANISM")
|
||||
for k, v in mechanism_counts.most_common():
|
||||
lines.append(f" {k:42s} {v}")
|
||||
lines.append("")
|
||||
lines.append("NEXT-STEP CLASS (表现分类，不是最终根因)")
|
||||
for k, v in next_counts.most_common():
|
||||
lines.append(f" {k:42s} {v}")
|
||||
lines.append("")
|
||||
lines.append("SHARED INVALID POINT GROUPS")
|
||||
lines.append(f" groups with >=2 invalid records: {len(shared_groups)}")
|
||||
lines.append("")
|
||||
lines.append("INTERPRETATION RULES")
|
||||
lines.append(
|
||||
" CELL_BOUNDARY_SHARED_WITH_COVERAGE / CELL_BOUNDARY_TOPOLOGY_CANDIDATE "
|
||||
"优先调查 Cell clipping / shared topology / importer ring assembly。"
|
||||
)
|
||||
lines.append(
|
||||
" INTERNAL_TRUE_SEGMENT_CROSSING 优先调查源拓扑与 importer edge/ring 顺序。"
|
||||
)
|
||||
lines.append(
|
||||
" INTERNAL_REPEATED_VERTEX_SELF_TOUCH 需要判断 CM93 是否允许 touch-ring 表达，"
|
||||
"以及 importer 是否应输出 MultiPolygon/拆环。"
|
||||
)
|
||||
lines.append(
|
||||
" 本脚本不执行 make_valid、buffer(0)、闭环、补点、删点或任何数据库写入。"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("PBF: OUT OF SCOPE / NOT STARTED")
|
||||
|
||||
(outdir / "invalid_area_audit_summary_v4.txt").write_text(
|
||||
"\n".join(lines),
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
conn.rollback()
|
||||
|
||||
print("[6/6] 压缩输出 ...")
|
||||
zpath = zip_dir(outdir)
|
||||
print("")
|
||||
print("完成。")
|
||||
print(f"Output: {outdir.resolve()}")
|
||||
print(f"ZIP: {zpath.resolve()}")
|
||||
print("请把 ZIP 整包发给 ChatGPT。")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
print("\n用户中断。")
|
||||
sys.exit(130)
|
||||
except Exception:
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
tb = traceback.format_exc()
|
||||
print(tb)
|
||||
try:
|
||||
(outdir / "fatal_error.txt").write_text(tb, encoding="utf-8")
|
||||
except Exception:
|
||||
pass
|
||||
sys.exit(1)
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user