1714 lines
50 KiB
Python
1714 lines
50 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
CM93 -> MySQL OGC Polygon Normalization Dry-Run
|
||
Version: v9.0
|
||
|
||
背景
|
||
====
|
||
前序审计已经确认：
|
||
|
||
1. Dataset ID=1 为日本沿海局部 CM93 数据，不是完整 CM93。
|
||
因此跨未导入海域的边界断开不作为错误。
|
||
|
||
2. 原来的 4406 条 geometry issue：
|
||
全部是 raw_geotype=16 / META 的正常 parent-no-geometry，
|
||
应分类为 EXPECTED_PARENT_NO_GEOMETRY，而非 geometry corruption。
|
||
|
||
3. 已实际生成的空间数据：
|
||
- Point invalid = 0
|
||
- Line invalid = 0
|
||
- Sounding invalid = 0
|
||
- Area OGC-invalid = 52
|
||
- Coverage OGC-invalid = 3
|
||
|
||
4. V6 已证明：
|
||
52/52 invalid Area 的 raw CM93 edge chain 本身就会生成同样的 invalid topology，
|
||
importer 没有发现 edge index / edge direction / edge ordering / transform 错误。
|
||
|
||
5. V8 已证明：
|
||
- 41 条记录存在 linear remainder；
|
||
- 共 109 条 remainder segment；
|
||
- 109/109 都是 OPPOSITE_DIRECTION_RETRACE；
|
||
- 即同一路径正向走一次、反向走一次的零面积 connector / bridge；
|
||
- 55/55 的 face selection 全部兼容 odd-even；
|
||
- 54/55 同时兼容 non-zero；
|
||
- 1 条只兼容 odd-even。
|
||
因此 odd-even 是当前证据下更安全的 CM93 Area fill rule。
|
||
|
||
V9 目的
|
||
=======
|
||
这是 normalization 的“干跑 / 验证器”，不是数据库修复器。
|
||
|
||
对 52 invalid Area + 3 invalid Coverage：
|
||
1. 对原始 boundary 进行 noding + polygonize_full；
|
||
2. 对每一个 planar face 取 representative point；
|
||
3. 按原始 ring 的 odd-even parity 决定该 face 是否属于 Area；
|
||
4. union 所有 odd-even inside faces；
|
||
5. 得到 candidate OGC-valid Polygon / MultiPolygon；
|
||
6. 只在内存中验证 candidate：
|
||
|
||
- candidate 是否 non-empty；
|
||
- candidate 是否只包含 Polygon/MultiPolygon；
|
||
- candidate 是否 valid；
|
||
- candidate 是否与 Shapely make_valid 的 polygonal 部分一致；
|
||
- candidate 是否没有产生原始 linework 之外的新边界；
|
||
- 原始 boundary 中被 candidate 丢弃的线是否与 make_valid linear remainder 一致；
|
||
- candidate 再次 normalization 是否幂等；
|
||
- candidate bbox 与原始 bbox 的变化；
|
||
- 生成稳定 SHA-256 WKB hash，方便后续重复验证。
|
||
|
||
7. 对全部“本来就 valid”的 Area/Coverage 做 policy guard：
|
||
normalization 函数遇到 valid geometry 必须 PASS-THROUGH，
|
||
不得改变任何有效 geometry。
|
||
本脚本会统计 valid record 数量并确认 policy_changed_count=0。
|
||
|
||
V9 不做什么
|
||
============
|
||
- 不 UPDATE
|
||
- 不 INSERT
|
||
- 不 DELETE
|
||
- 不 ST_MakeValid 写回
|
||
- 不替换 chart_area_geometry
|
||
- 不替换 chart_coverage
|
||
- 不修改 geometry_issue
|
||
- 不重新导入
|
||
- 不开始 PBF
|
||
|
||
V9 的 PASS 含义
|
||
===============
|
||
只有当 55 条全部满足：
|
||
|
||
- odd-even candidate 成功；
|
||
- candidate valid；
|
||
- candidate polygonal-only；
|
||
- candidate 与 make_valid polygonal 部分面积拓扑一致；
|
||
- candidate 没有引入原始 boundary 之外的新线；
|
||
- discarded linework 与已确认 retraced remainder 一致（若有）；
|
||
- normalization 幂等；
|
||
|
||
才允许下一阶段讨论 importer/schema 如何保存 normalized geometry。
|
||
|
||
即使 V9 全 PASS，本脚本也绝不写数据库。
|
||
|
||
依赖
|
||
====
|
||
pip install pymysql shapely
|
||
|
||
执行
|
||
====
|
||
python cm93_mysql_normalization_dryrun_v9.py ^
|
||
--host 127.0.0.1 ^
|
||
--port 3306 ^
|
||
--user root ^
|
||
--database cm93_chart ^
|
||
--dataset-id 1
|
||
|
||
输出
|
||
====
|
||
- normalization_dryrun_summary_v9.txt
|
||
- normalization_dryrun_report_v9.json
|
||
- normalization_candidate_results.csv
|
||
- normalization_face_selection.csv
|
||
- normalization_boundary_compare.csv
|
||
- normalization_failure_detail.csv
|
||
- normalization_by_object_class.csv
|
||
- normalization_by_boundary_context.csv
|
||
- normalization_candidates.geojson
|
||
并自动生成 ZIP。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import getpass
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import os
|
||
import re
|
||
import sys
|
||
import traceback
|
||
import zipfile
|
||
from collections import Counter
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||
|
||
try:
|
||
import pymysql
|
||
from pymysql.cursors import DictCursor
|
||
except ImportError:
|
||
print("缺少 pymysql：pip install pymysql shapely")
|
||
sys.exit(2)
|
||
|
||
try:
|
||
import shapely
|
||
|
||
try:
|
||
from shapely import from_wkb as _from_wkb
|
||
def load_wkb(data: bytes):
|
||
return _from_wkb(data)
|
||
except ImportError:
|
||
from shapely import wkb as _wkb
|
||
def load_wkb(data: bytes):
|
||
return _wkb.loads(data)
|
||
|
||
try:
|
||
from shapely import to_wkb as _to_wkb
|
||
def dump_wkb(g) -> bytes:
|
||
return bytes(_to_wkb(g, hex=False, byte_order=1))
|
||
except ImportError:
|
||
from shapely import wkb as _wkb2
|
||
def dump_wkb(g) -> bytes:
|
||
return _wkb2.dumps(g, hex=False, big_endian=False)
|
||
|
||
try:
|
||
from shapely import make_valid as _make_valid
|
||
def make_valid_geom(g):
|
||
return _make_valid(g)
|
||
except ImportError:
|
||
from shapely.validation import make_valid as _make_valid
|
||
def make_valid_geom(g):
|
||
return _make_valid(g)
|
||
|
||
from shapely.geometry import Point, mapping
|
||
from shapely.ops import unary_union, polygonize_full
|
||
from shapely.validation import explain_validity
|
||
|
||
except ImportError:
|
||
print("缺少 shapely：pip install pymysql shapely")
|
||
sys.exit(2)
|
||
|
||
|
||
INVALID_POINT_RE = re.compile(
|
||
r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+"
|
||
r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Basic helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any:
|
||
cur.execute(sql, params)
|
||
row = cur.fetchone()
|
||
return next(iter(row.values())) if row else None
|
||
|
||
|
||
def begin_read_only(cur, warnings: List[str]):
|
||
try:
|
||
cur.execute("SET SESSION TRANSACTION READ ONLY")
|
||
except Exception as e:
|
||
warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}")
|
||
|
||
try:
|
||
cur.execute("START TRANSACTION READ ONLY")
|
||
except Exception as e:
|
||
warnings.append(f"START TRANSACTION READ ONLY failed: {e}")
|
||
cur.execute("START TRANSACTION")
|
||
|
||
|
||
def to_bytes(v: Any) -> Optional[bytes]:
|
||
if v is None:
|
||
return None
|
||
if isinstance(v, bytes):
|
||
return v
|
||
if isinstance(v, bytearray):
|
||
return bytes(v)
|
||
if isinstance(v, memoryview):
|
||
return v.tobytes()
|
||
try:
|
||
return bytes(v)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def write_csv(
|
||
path: Path,
|
||
rows: Iterable[Dict[str, Any]],
|
||
fieldnames: Optional[List[str]] = None,
|
||
):
|
||
rows = list(rows)
|
||
|
||
if fieldnames is None:
|
||
seen = set()
|
||
fieldnames = []
|
||
for row in rows:
|
||
for k in row.keys():
|
||
if k not in seen:
|
||
seen.add(k)
|
||
fieldnames.append(k)
|
||
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||
if not fieldnames:
|
||
return
|
||
|
||
writer = csv.DictWriter(
|
||
f,
|
||
fieldnames=fieldnames,
|
||
extrasaction="ignore"
|
||
)
|
||
writer.writeheader()
|
||
|
||
for row in rows:
|
||
out = {}
|
||
for k in fieldnames:
|
||
v = row.get(k)
|
||
if isinstance(v, (dict, list, tuple)):
|
||
v = json.dumps(v, ensure_ascii=False, default=str)
|
||
out[k] = "" if v is None else v
|
||
writer.writerow(out)
|
||
|
||
|
||
def probe_wkb_fn(cur) -> str:
|
||
for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"):
|
||
try:
|
||
cur.execute(
|
||
f"SELECT {fn}(geom) AS x "
|
||
f"FROM chart_area_geometry LIMIT 1"
|
||
)
|
||
cur.fetchone()
|
||
return fn
|
||
except Exception:
|
||
pass
|
||
|
||
raise RuntimeError("找不到 geometry -> WKB 函数")
|
||
|
||
|
||
def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]:
|
||
if not reason:
|
||
return None
|
||
|
||
m = INVALID_POINT_RE.search(reason)
|
||
if not m:
|
||
return None
|
||
|
||
try:
|
||
return float(m.group(1)), float(m.group(2))
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def bbox_from_row(row: Dict[str, Any]) -> Optional[Tuple[float, float, float, float]]:
|
||
try:
|
||
vals = (
|
||
float(row["cell_bbox_min_lon"]),
|
||
float(row["cell_bbox_min_lat"]),
|
||
float(row["cell_bbox_max_lon"]),
|
||
float(row["cell_bbox_max_lat"]),
|
||
)
|
||
if all(math.isfinite(v) for v in vals):
|
||
return vals
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
|
||
def invalid_point_cell_edge(
|
||
p: Optional[Tuple[float, float]],
|
||
bbox: Optional[Tuple[float, float, float, float]],
|
||
tol: float,
|
||
) -> Tuple[int, str, Optional[float]]:
|
||
if p is None or bbox is None:
|
||
return 0, "", None
|
||
|
||
x, y = p
|
||
minx, miny, maxx, maxy = bbox
|
||
|
||
d = {
|
||
"WEST": abs(x - minx),
|
||
"EAST": abs(x - maxx),
|
||
"SOUTH": abs(y - miny),
|
||
"NORTH": abs(y - maxy),
|
||
}
|
||
|
||
edge, dist = min(d.items(), key=lambda kv: kv[1])
|
||
return int(dist <= tol), edge if dist <= tol else "", dist
|
||
|
||
|
||
def geometry_hash(g) -> Optional[str]:
|
||
if g is None:
|
||
return None
|
||
try:
|
||
return hashlib.sha256(dump_wkb(g)).hexdigest()
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def safe_area(g) -> Optional[float]:
|
||
if g is None:
|
||
return None
|
||
try:
|
||
return float(g.area)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def safe_length(g) -> Optional[float]:
|
||
if g is None:
|
||
return None
|
||
try:
|
||
return float(g.length)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def safe_symdiff_area(a, b) -> Optional[float]:
|
||
if a is None or b is None:
|
||
return None
|
||
try:
|
||
return float(a.symmetric_difference(b).area)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def safe_symdiff_length(a, b) -> Optional[float]:
|
||
if a is None or b is None:
|
||
return None
|
||
try:
|
||
return float(a.symmetric_difference(b).length)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def safe_hausdorff(a, b) -> Optional[float]:
|
||
if a is None or b is None:
|
||
return None
|
||
try:
|
||
return float(a.hausdorff_distance(b))
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def flatten_components(g) -> Dict[str, List[Any]]:
|
||
result = {
|
||
"polygons": [],
|
||
"lines": [],
|
||
"points": [],
|
||
"others": [],
|
||
}
|
||
|
||
def rec(x):
|
||
if x is None:
|
||
return
|
||
|
||
t = x.geom_type
|
||
|
||
if t == "Polygon":
|
||
result["polygons"].append(x)
|
||
elif t == "MultiPolygon":
|
||
for y in x.geoms:
|
||
rec(y)
|
||
elif t in ("LineString", "LinearRing"):
|
||
result["lines"].append(x)
|
||
elif t == "MultiLineString":
|
||
for y in x.geoms:
|
||
rec(y)
|
||
elif t == "Point":
|
||
result["points"].append(x)
|
||
elif t == "MultiPoint":
|
||
for y in x.geoms:
|
||
rec(y)
|
||
elif hasattr(x, "geoms"):
|
||
for y in x.geoms:
|
||
rec(y)
|
||
else:
|
||
result["others"].append(x)
|
||
|
||
rec(g)
|
||
return result
|
||
|
||
|
||
def geom_union(items: Sequence[Any]) -> Optional[Any]:
|
||
if not items:
|
||
return None
|
||
|
||
try:
|
||
return unary_union(list(items))
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Odd-even fill rule
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def original_rings(geom) -> List[List[Tuple[float, float]]]:
|
||
rings: List[List[Tuple[float, float]]] = []
|
||
|
||
def add_polygon(poly):
|
||
rings.append([
|
||
(float(c[0]), float(c[1]))
|
||
for c in poly.exterior.coords
|
||
])
|
||
|
||
for ring in poly.interiors:
|
||
rings.append([
|
||
(float(c[0]), float(c[1]))
|
||
for c in ring.coords
|
||
])
|
||
|
||
if geom.geom_type == "Polygon":
|
||
add_polygon(geom)
|
||
elif geom.geom_type == "MultiPolygon":
|
||
for poly in geom.geoms:
|
||
add_polygon(poly)
|
||
|
||
return rings
|
||
|
||
|
||
def point_on_segment(
|
||
px: float,
|
||
py: float,
|
||
ax: float,
|
||
ay: float,
|
||
bx: float,
|
||
by: float,
|
||
eps: float = 1e-12,
|
||
) -> bool:
|
||
vx = bx - ax
|
||
vy = by - ay
|
||
wx = px - ax
|
||
wy = py - ay
|
||
|
||
cross_val = abs(vx * wy - vy * wx)
|
||
if cross_val > eps * max(1.0, abs(vx), abs(vy)):
|
||
return False
|
||
|
||
dot_val = wx * vx + wy * vy
|
||
if dot_val < -eps:
|
||
return False
|
||
|
||
vv = vx * vx + vy * vy
|
||
if dot_val - vv > eps:
|
||
return False
|
||
|
||
return True
|
||
|
||
|
||
def even_odd_inside(
|
||
point: Tuple[float, float],
|
||
rings: List[List[Tuple[float, float]]],
|
||
) -> bool:
|
||
x, y = point
|
||
crossings = 0
|
||
|
||
for ring in rings:
|
||
for i in range(len(ring) - 1):
|
||
x1, y1 = ring[i]
|
||
x2, y2 = ring[i + 1]
|
||
|
||
if point_on_segment(x, y, x1, y1, x2, y2):
|
||
# representative point 理论上不应落在 face boundary。
|
||
# 若数值误差导致落边界，按 inside 处理。
|
||
return True
|
||
|
||
if (y1 > y) != (y2 > y):
|
||
xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1)
|
||
if xinters > x:
|
||
crossings += 1
|
||
|
||
return (crossings % 2) == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Candidate normalization
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def odd_even_normalize_invalid(
|
||
geom
|
||
) -> Tuple[Optional[Any], Dict[str, Any], List[Dict[str, Any]]]:
|
||
"""
|
||
对 invalid Polygon/MultiPolygon：
|
||
|
||
1. noded boundary
|
||
2. polygonize_full
|
||
3. face representative point
|
||
4. odd-even selection
|
||
5. union selected faces
|
||
|
||
不调用 make_valid 生成 candidate。
|
||
make_valid 只在外部作为独立对照参考。
|
||
"""
|
||
info: Dict[str, Any] = {
|
||
"polygonize_face_count": 0,
|
||
"selected_face_count": 0,
|
||
"cuts_length_deg": None,
|
||
"dangles_length_deg": None,
|
||
"invalid_rings_length_deg": None,
|
||
"error": None,
|
||
}
|
||
|
||
face_rows: List[Dict[str, Any]] = []
|
||
|
||
try:
|
||
rings = original_rings(geom)
|
||
noded_boundary = unary_union(geom.boundary)
|
||
|
||
polygons_geom, cuts, dangles, invalid_rings = polygonize_full(
|
||
noded_boundary
|
||
)
|
||
|
||
face_components = flatten_components(polygons_geom)["polygons"]
|
||
|
||
info["polygonize_face_count"] = len(face_components)
|
||
info["cuts_length_deg"] = safe_length(cuts)
|
||
info["dangles_length_deg"] = safe_length(dangles)
|
||
info["invalid_rings_length_deg"] = safe_length(invalid_rings)
|
||
|
||
selected = []
|
||
|
||
for idx, face in enumerate(face_components):
|
||
rp = face.representative_point()
|
||
p = (float(rp.x), float(rp.y))
|
||
|
||
inside = even_odd_inside(p, rings)
|
||
|
||
face_rows.append({
|
||
"face_index": idx,
|
||
"representative_x": p[0],
|
||
"representative_y": p[1],
|
||
"face_area_deg2": float(face.area),
|
||
"odd_even_inside": int(inside),
|
||
})
|
||
|
||
if inside:
|
||
selected.append(face)
|
||
|
||
info["selected_face_count"] = len(selected)
|
||
|
||
if not selected:
|
||
return None, info, face_rows
|
||
|
||
candidate = unary_union(selected)
|
||
|
||
return candidate, info, face_rows
|
||
|
||
except Exception as e:
|
||
info["error"] = f"{type(e).__name__}: {e}"
|
||
return None, info, face_rows
|
||
|
||
|
||
def normalize_policy(geom) -> Tuple[Any, str]:
|
||
"""
|
||
最终候选 policy 的核心行为：
|
||
|
||
- valid geometry: 原样 PASS-THROUGH
|
||
- invalid polygonal geometry: odd-even normalization
|
||
|
||
这里只用于 dry-run。
|
||
"""
|
||
if geom.is_valid:
|
||
return geom, "PASS_THROUGH_VALID"
|
||
|
||
candidate, info, _ = odd_even_normalize_invalid(geom)
|
||
|
||
if candidate is None:
|
||
return geom, "NORMALIZATION_FAILED_RETURN_ORIGINAL"
|
||
|
||
return candidate, "ODD_EVEN_NORMALIZED"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Database loader
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def load_polygon_records(
|
||
cur,
|
||
dataset_id: int,
|
||
wkb_fn: str
|
||
) -> List[Dict[str, Any]]:
|
||
records = []
|
||
|
||
cur.execute(f"""
|
||
SELECT
|
||
'AREA' AS kind,
|
||
f.id AS feature_id,
|
||
NULL AS coverage_geom_id,
|
||
f.object_class,
|
||
f.cell_id,
|
||
c.cell_name,
|
||
f.source_record_index,
|
||
f.coverage_id,
|
||
f.raw_geotype,
|
||
f.bbox_min_lon,
|
||
f.bbox_min_lat,
|
||
f.bbox_max_lon,
|
||
f.bbox_max_lat,
|
||
c.bbox_min_lon AS cell_bbox_min_lon,
|
||
c.bbox_min_lat AS cell_bbox_min_lat,
|
||
c.bbox_max_lon AS cell_bbox_max_lon,
|
||
c.bbox_max_lat AS cell_bbox_max_lat,
|
||
{wkb_fn}(g.geom) AS geom_wkb
|
||
FROM chart_feature f
|
||
JOIN chart_area_geometry g
|
||
ON g.feature_id = f.id
|
||
JOIN chart_cell c
|
||
ON c.id = f.cell_id
|
||
WHERE f.dataset_id=%s
|
||
AND f.geometry_type='AREA'
|
||
ORDER BY f.id
|
||
""", (dataset_id,))
|
||
|
||
for r in cur:
|
||
try:
|
||
geom = load_wkb(to_bytes(r["geom_wkb"]))
|
||
x = dict(r)
|
||
x["_geom"] = geom
|
||
records.append(x)
|
||
except Exception:
|
||
continue
|
||
|
||
cur.execute(f"""
|
||
SELECT
|
||
'COVERAGE' AS kind,
|
||
NULL AS feature_id,
|
||
v.id AS coverage_geom_id,
|
||
'COVERAGE' AS object_class,
|
||
v.cell_id,
|
||
c.cell_name,
|
||
v.source_record_index,
|
||
NULL AS coverage_id,
|
||
NULL AS raw_geotype,
|
||
NULL AS bbox_min_lon,
|
||
NULL AS bbox_min_lat,
|
||
NULL AS bbox_max_lon,
|
||
NULL AS bbox_max_lat,
|
||
c.bbox_min_lon AS cell_bbox_min_lon,
|
||
c.bbox_min_lat AS cell_bbox_min_lat,
|
||
c.bbox_max_lon AS cell_bbox_max_lon,
|
||
c.bbox_max_lat AS cell_bbox_max_lat,
|
||
{wkb_fn}(v.geom) AS geom_wkb
|
||
FROM chart_coverage v
|
||
JOIN chart_cell c
|
||
ON c.id = v.cell_id
|
||
WHERE v.dataset_id=%s
|
||
ORDER BY v.id
|
||
""", (dataset_id,))
|
||
|
||
for r in cur:
|
||
try:
|
||
geom = load_wkb(to_bytes(r["geom_wkb"]))
|
||
x = dict(r)
|
||
x["_geom"] = geom
|
||
records.append(x)
|
||
except Exception:
|
||
continue
|
||
|
||
return records
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# GeoJSON
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def make_geojson(records: List[Dict[str, Any]], path: Path):
|
||
features = []
|
||
|
||
def add(g, props):
|
||
if g is None:
|
||
return
|
||
|
||
try:
|
||
features.append({
|
||
"type": "Feature",
|
||
"geometry": mapping(g),
|
||
"properties": props,
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
for r in records:
|
||
if r["_geom"].is_valid:
|
||
continue
|
||
|
||
rid = (
|
||
r.get("feature_id")
|
||
if r["kind"] == "AREA"
|
||
else r.get("coverage_geom_id")
|
||
)
|
||
|
||
common = {
|
||
"kind": r["kind"],
|
||
"record_id": rid,
|
||
"object_class": r.get("object_class"),
|
||
"cell_name": r.get("cell_name"),
|
||
"status": r.get("_candidate_status"),
|
||
}
|
||
|
||
add(
|
||
r.get("_geom"),
|
||
{
|
||
**common,
|
||
"layer": "original_invalid",
|
||
}
|
||
)
|
||
|
||
add(
|
||
r.get("_candidate"),
|
||
{
|
||
**common,
|
||
"layer": "odd_even_candidate",
|
||
}
|
||
)
|
||
|
||
add(
|
||
r.get("_discarded_linework"),
|
||
{
|
||
**common,
|
||
"layer": "discarded_linework",
|
||
}
|
||
)
|
||
|
||
p = r.get("_invalid_point")
|
||
if p is not None:
|
||
add(
|
||
Point(p),
|
||
{
|
||
**common,
|
||
"layer": "invalidity_point",
|
||
}
|
||
)
|
||
|
||
path.write_text(
|
||
json.dumps(
|
||
{
|
||
"type": "FeatureCollection",
|
||
"features": features,
|
||
},
|
||
ensure_ascii=False,
|
||
separators=(",", ":"),
|
||
default=str,
|
||
),
|
||
encoding="utf-8"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(
|
||
description="CM93 OGC polygon normalization dry-run v9"
|
||
)
|
||
|
||
parser.add_argument(
|
||
"--host",
|
||
default=os.getenv("MYSQL_HOST", "127.0.0.1")
|
||
)
|
||
parser.add_argument(
|
||
"--port",
|
||
type=int,
|
||
default=int(os.getenv("MYSQL_PORT", "3306"))
|
||
)
|
||
parser.add_argument(
|
||
"--user",
|
||
default=os.getenv("MYSQL_USER", "root")
|
||
)
|
||
parser.add_argument(
|
||
"--password",
|
||
default=os.getenv("MYSQL_PASSWORD")
|
||
)
|
||
parser.add_argument(
|
||
"--database",
|
||
default=os.getenv("MYSQL_DATABASE", "cm93_chart")
|
||
)
|
||
parser.add_argument(
|
||
"--dataset-id",
|
||
type=int,
|
||
default=1
|
||
)
|
||
parser.add_argument(
|
||
"--cell-edge-tolerance",
|
||
type=float,
|
||
default=5e-6
|
||
)
|
||
parser.add_argument(
|
||
"--area-abs-tolerance",
|
||
type=float,
|
||
default=1e-12,
|
||
help="candidate vs make_valid polygonal symmetric-difference area absolute tolerance"
|
||
)
|
||
parser.add_argument(
|
||
"--area-relative-tolerance",
|
||
type=float,
|
||
default=1e-6
|
||
)
|
||
parser.add_argument(
|
||
"--line-length-tolerance",
|
||
type=float,
|
||
default=1e-10,
|
||
help="boundary linework compare tolerance (degree)"
|
||
)
|
||
parser.add_argument(
|
||
"--output",
|
||
default=None
|
||
)
|
||
|
||
return parser.parse_args()
|
||
|
||
|
||
def main():
|
||
args = parse_args()
|
||
|
||
if not args.password:
|
||
args.password = getpass.getpass("MySQL password: ")
|
||
|
||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
|
||
outdir = Path(
|
||
args.output
|
||
or f"cm93_normalization_dryrun_v9_dataset_{args.dataset_id}_{stamp}"
|
||
)
|
||
outdir.mkdir(parents=True, exist_ok=True)
|
||
|
||
report: Dict[str, Any] = {
|
||
"version": "v9.0",
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"database": args.database,
|
||
"dataset_id": args.dataset_id,
|
||
"read_only": True,
|
||
"pbf": "OUT_OF_SCOPE",
|
||
"shapely_version": getattr(shapely, "__version__", "unknown"),
|
||
"scope_note": (
|
||
"CM93JAPAN 是日本沿海局部 CM93 数据集。"
|
||
"V9 只验证 odd-even OGC normalization candidate，"
|
||
"不修改数据库。"
|
||
),
|
||
"warnings": [],
|
||
}
|
||
|
||
conn = None
|
||
|
||
try:
|
||
conn = pymysql.connect(
|
||
host=args.host,
|
||
port=args.port,
|
||
user=args.user,
|
||
password=args.password,
|
||
database=args.database,
|
||
charset="utf8mb4",
|
||
cursorclass=DictCursor,
|
||
autocommit=False,
|
||
connect_timeout=20,
|
||
read_timeout=600,
|
||
write_timeout=30,
|
||
)
|
||
|
||
with conn.cursor() as cur:
|
||
begin_read_only(cur, report["warnings"])
|
||
|
||
report["mysql_version"] = scalar(
|
||
cur,
|
||
"SELECT VERSION()"
|
||
)
|
||
|
||
wkb_fn = probe_wkb_fn(cur)
|
||
report["wkb_fn"] = wkb_fn
|
||
|
||
print("[1/6] 加载全部 Area + Coverage ...")
|
||
records = load_polygon_records(
|
||
cur,
|
||
args.dataset_id,
|
||
wkb_fn
|
||
)
|
||
|
||
area_count = sum(r["kind"] == "AREA" for r in records)
|
||
coverage_count = sum(r["kind"] == "COVERAGE" for r in records)
|
||
|
||
valid_records = [r for r in records if r["_geom"].is_valid]
|
||
invalid_records = [r for r in records if not r["_geom"].is_valid]
|
||
|
||
print(
|
||
f" Area={area_count:,} Coverage={coverage_count:,} "
|
||
f"valid={len(valid_records):,} invalid={len(invalid_records):,}"
|
||
)
|
||
|
||
# ----------------------------------------------------------------
|
||
# Valid geometry policy guard
|
||
# ----------------------------------------------------------------
|
||
|
||
print("[2/6] Valid geometry PASS-THROUGH guard ...")
|
||
|
||
valid_policy_changed = 0
|
||
valid_policy_hash_changed = 0
|
||
|
||
for idx, r in enumerate(valid_records, start=1):
|
||
geom = r["_geom"]
|
||
|
||
normalized, mode = normalize_policy(geom)
|
||
|
||
if mode != "PASS_THROUGH_VALID":
|
||
valid_policy_changed += 1
|
||
|
||
if geometry_hash(normalized) != geometry_hash(geom):
|
||
valid_policy_hash_changed += 1
|
||
|
||
if idx % 20000 == 0:
|
||
print(
|
||
f" valid checked={idx:,}/{len(valid_records):,}"
|
||
)
|
||
|
||
print(
|
||
f" valid policy changed={valid_policy_changed} "
|
||
f"hash_changed={valid_policy_hash_changed}"
|
||
)
|
||
|
||
# ----------------------------------------------------------------
|
||
# Invalid dry-run
|
||
# ----------------------------------------------------------------
|
||
|
||
print("[3/6] 55 invalid Polygon odd-even normalization dry-run ...")
|
||
|
||
candidate_rows: List[Dict[str, Any]] = []
|
||
face_rows: List[Dict[str, Any]] = []
|
||
boundary_rows: List[Dict[str, Any]] = []
|
||
failure_rows: List[Dict[str, Any]] = []
|
||
|
||
for idx, r in enumerate(invalid_records, start=1):
|
||
geom = r["_geom"]
|
||
|
||
rid = (
|
||
r.get("feature_id")
|
||
if r["kind"] == "AREA"
|
||
else r.get("coverage_geom_id")
|
||
)
|
||
|
||
reason = explain_validity(geom)
|
||
invalid_point = parse_invalid_point(reason)
|
||
|
||
cell_bbox = bbox_from_row(r)
|
||
|
||
on_cell_edge, cell_edge, cell_edge_distance = (
|
||
invalid_point_cell_edge(
|
||
invalid_point,
|
||
cell_bbox,
|
||
args.cell_edge_tolerance
|
||
)
|
||
)
|
||
|
||
candidate, norm_info, this_faces = (
|
||
odd_even_normalize_invalid(geom)
|
||
)
|
||
|
||
base = {
|
||
"kind": r["kind"],
|
||
"feature_id": r.get("feature_id"),
|
||
"coverage_geom_id": r.get("coverage_geom_id"),
|
||
"object_class": r.get("object_class"),
|
||
"cell_id": r.get("cell_id"),
|
||
"cell_name": r.get("cell_name"),
|
||
"source_record_index": r.get("source_record_index"),
|
||
"coverage_id": r.get("coverage_id"),
|
||
"raw_geotype": r.get("raw_geotype"),
|
||
"original_validity_reason": reason,
|
||
"invalid_x": (
|
||
invalid_point[0]
|
||
if invalid_point else None
|
||
),
|
||
"invalid_y": (
|
||
invalid_point[1]
|
||
if invalid_point else None
|
||
),
|
||
"invalid_point_on_cell_edge": on_cell_edge,
|
||
"cell_edge": cell_edge,
|
||
"cell_edge_distance_deg": cell_edge_distance,
|
||
}
|
||
|
||
for face in this_faces:
|
||
face_rows.append({
|
||
**base,
|
||
**face,
|
||
})
|
||
|
||
if candidate is None:
|
||
row = {
|
||
**base,
|
||
**norm_info,
|
||
"candidate_status": "FAIL_NO_CANDIDATE",
|
||
"candidate_type": None,
|
||
"candidate_valid": 0,
|
||
}
|
||
|
||
candidate_rows.append(row)
|
||
failure_rows.append(row)
|
||
|
||
r["_candidate"] = None
|
||
r["_candidate_status"] = "FAIL_NO_CANDIDATE"
|
||
continue
|
||
|
||
# candidate must be polygonal-only
|
||
candidate_comps = flatten_components(candidate)
|
||
|
||
candidate_polygon_only = (
|
||
len(candidate_comps["polygons"]) > 0
|
||
and len(candidate_comps["lines"]) == 0
|
||
and len(candidate_comps["points"]) == 0
|
||
and len(candidate_comps["others"]) == 0
|
||
)
|
||
|
||
candidate_valid = bool(candidate.is_valid)
|
||
candidate_empty = bool(candidate.is_empty)
|
||
|
||
# independent make_valid reference
|
||
mv = make_valid_geom(geom)
|
||
mv_comps = flatten_components(mv)
|
||
mv_poly_union = geom_union(mv_comps["polygons"])
|
||
mv_line_union = geom_union(mv_comps["lines"])
|
||
|
||
candidate_mv_symarea = safe_symdiff_area(
|
||
candidate,
|
||
mv_poly_union
|
||
)
|
||
|
||
candidate_area = safe_area(candidate)
|
||
mv_area = safe_area(mv_poly_union)
|
||
|
||
if (
|
||
candidate_mv_symarea is not None
|
||
and candidate_area is not None
|
||
and mv_area is not None
|
||
):
|
||
area_scale = max(
|
||
abs(candidate_area),
|
||
abs(mv_area),
|
||
1e-30
|
||
)
|
||
|
||
area_allowed = max(
|
||
args.area_abs_tolerance,
|
||
args.area_relative_tolerance * area_scale
|
||
)
|
||
|
||
area_match = (
|
||
candidate_mv_symarea <= area_allowed
|
||
)
|
||
else:
|
||
area_allowed = None
|
||
area_match = False
|
||
|
||
# Boundary evidence:
|
||
# original linework is noded before comparison.
|
||
try:
|
||
original_noded_boundary = unary_union(
|
||
geom.boundary
|
||
)
|
||
except Exception:
|
||
original_noded_boundary = geom.boundary
|
||
|
||
try:
|
||
candidate_boundary = unary_union(
|
||
candidate.boundary
|
||
)
|
||
except Exception:
|
||
candidate_boundary = candidate.boundary
|
||
|
||
# candidate must not invent boundary outside original linework
|
||
try:
|
||
candidate_extra_boundary = (
|
||
candidate_boundary.difference(
|
||
original_noded_boundary
|
||
)
|
||
)
|
||
except Exception:
|
||
candidate_extra_boundary = None
|
||
|
||
candidate_extra_length = safe_length(
|
||
candidate_extra_boundary
|
||
)
|
||
|
||
no_new_boundary = (
|
||
candidate_extra_length is not None
|
||
and candidate_extra_length <= args.line_length_tolerance
|
||
)
|
||
|
||
# linework removed from polygon boundary
|
||
try:
|
||
discarded_linework = (
|
||
original_noded_boundary.difference(
|
||
candidate_boundary
|
||
)
|
||
)
|
||
except Exception:
|
||
discarded_linework = None
|
||
|
||
discarded_length = safe_length(discarded_linework)
|
||
mv_remainder_length = safe_length(mv_line_union)
|
||
|
||
discarded_vs_mv_sym_length = safe_symdiff_length(
|
||
discarded_linework,
|
||
mv_line_union
|
||
)
|
||
|
||
if mv_line_union is None:
|
||
discarded_matches_mv = (
|
||
(discarded_length or 0.0)
|
||
<= args.line_length_tolerance
|
||
)
|
||
else:
|
||
discarded_matches_mv = (
|
||
discarded_vs_mv_sym_length is not None
|
||
and discarded_vs_mv_sym_length
|
||
<= args.line_length_tolerance
|
||
)
|
||
|
||
discarded_mv_hausdorff = safe_hausdorff(
|
||
discarded_linework,
|
||
mv_line_union
|
||
)
|
||
|
||
# Idempotence:
|
||
candidate2, mode2 = normalize_policy(candidate)
|
||
|
||
candidate_idempotent_symarea = safe_symdiff_area(
|
||
candidate,
|
||
candidate2
|
||
)
|
||
|
||
idempotent = (
|
||
candidate2.is_valid
|
||
and candidate_idempotent_symarea is not None
|
||
and candidate_idempotent_symarea
|
||
<= args.area_abs_tolerance
|
||
and mode2 == "PASS_THROUGH_VALID"
|
||
)
|
||
|
||
# bbox
|
||
original_bounds = tuple(
|
||
float(v) for v in geom.bounds
|
||
)
|
||
candidate_bounds = tuple(
|
||
float(v) for v in candidate.bounds
|
||
)
|
||
|
||
bbox_changed = (
|
||
any(
|
||
abs(a - b) > args.cell_edge_tolerance
|
||
for a, b in zip(
|
||
original_bounds,
|
||
candidate_bounds
|
||
)
|
||
)
|
||
)
|
||
|
||
# Final PASS gate
|
||
pass_gate = all([
|
||
not candidate_empty,
|
||
candidate_polygon_only,
|
||
candidate_valid,
|
||
area_match,
|
||
no_new_boundary,
|
||
discarded_matches_mv,
|
||
idempotent,
|
||
])
|
||
|
||
candidate_status = (
|
||
"PASS_NORMALIZATION_CANDIDATE"
|
||
if pass_gate
|
||
else "CHECK_NORMALIZATION_CANDIDATE"
|
||
)
|
||
|
||
row = {
|
||
**base,
|
||
**norm_info,
|
||
|
||
"candidate_status": candidate_status,
|
||
"candidate_type": candidate.geom_type,
|
||
"candidate_empty": int(candidate_empty),
|
||
"candidate_valid": int(candidate_valid),
|
||
"candidate_validity_reason": (
|
||
"Valid Geometry"
|
||
if candidate_valid
|
||
else explain_validity(candidate)
|
||
),
|
||
"candidate_polygon_only": int(candidate_polygon_only),
|
||
|
||
"candidate_polygon_part_count":
|
||
len(candidate_comps["polygons"]),
|
||
"candidate_line_part_count":
|
||
len(candidate_comps["lines"]),
|
||
"candidate_point_part_count":
|
||
len(candidate_comps["points"]),
|
||
"candidate_other_part_count":
|
||
len(candidate_comps["others"]),
|
||
|
||
"candidate_area_deg2": candidate_area,
|
||
"make_valid_polygonal_area_deg2": mv_area,
|
||
"candidate_vs_make_valid_symdiff_area_deg2":
|
||
candidate_mv_symarea,
|
||
"candidate_make_valid_area_allowed_deg2":
|
||
area_allowed,
|
||
"candidate_matches_make_valid_polygonal":
|
||
int(area_match),
|
||
|
||
"candidate_extra_boundary_length_deg":
|
||
candidate_extra_length,
|
||
"candidate_introduces_no_new_boundary":
|
||
int(no_new_boundary),
|
||
|
||
"discarded_linework_length_deg":
|
||
discarded_length,
|
||
"make_valid_linear_remainder_length_deg":
|
||
mv_remainder_length,
|
||
"discarded_vs_make_valid_symdiff_length_deg":
|
||
discarded_vs_mv_sym_length,
|
||
"discarded_vs_make_valid_hausdorff_deg":
|
||
discarded_mv_hausdorff,
|
||
"discarded_matches_make_valid_remainder":
|
||
int(discarded_matches_mv),
|
||
|
||
"normalization_idempotent": int(idempotent),
|
||
"idempotent_symdiff_area_deg2":
|
||
candidate_idempotent_symarea,
|
||
|
||
"original_bounds": original_bounds,
|
||
"candidate_bounds": candidate_bounds,
|
||
"bbox_changed_over_tolerance":
|
||
int(bbox_changed),
|
||
|
||
"original_wkb_sha256":
|
||
geometry_hash(geom),
|
||
"candidate_wkb_sha256":
|
||
geometry_hash(candidate),
|
||
}
|
||
|
||
candidate_rows.append(row)
|
||
|
||
boundary_rows.append({
|
||
**base,
|
||
"candidate_status": candidate_status,
|
||
"original_boundary_length_deg":
|
||
safe_length(original_noded_boundary),
|
||
"candidate_boundary_length_deg":
|
||
safe_length(candidate_boundary),
|
||
"candidate_extra_boundary_length_deg":
|
||
candidate_extra_length,
|
||
"discarded_linework_length_deg":
|
||
discarded_length,
|
||
"make_valid_linear_remainder_length_deg":
|
||
mv_remainder_length,
|
||
"discarded_vs_make_valid_symdiff_length_deg":
|
||
discarded_vs_mv_sym_length,
|
||
"discarded_vs_make_valid_hausdorff_deg":
|
||
discarded_mv_hausdorff,
|
||
})
|
||
|
||
if not pass_gate:
|
||
failure_rows.append(row)
|
||
|
||
r["_candidate"] = candidate
|
||
r["_discarded_linework"] = discarded_linework
|
||
r["_candidate_status"] = candidate_status
|
||
r["_invalid_point"] = invalid_point
|
||
|
||
print(
|
||
f" [{idx:02d}/{len(invalid_records)}] "
|
||
f"{r['kind']} id={rid} "
|
||
f"{r.get('object_class')} -> "
|
||
f"{candidate_status}"
|
||
)
|
||
|
||
# ----------------------------------------------------------------
|
||
# Summary
|
||
# ----------------------------------------------------------------
|
||
|
||
print("[4/6] 汇总 PASS gate ...")
|
||
|
||
status_counts = Counter(
|
||
r["candidate_status"]
|
||
for r in candidate_rows
|
||
)
|
||
|
||
object_counts = Counter(
|
||
(
|
||
r["object_class"],
|
||
r["candidate_status"],
|
||
)
|
||
for r in candidate_rows
|
||
)
|
||
|
||
boundary_context_counts = Counter(
|
||
(
|
||
"CELL_EDGE"
|
||
if r["invalid_point_on_cell_edge"]
|
||
else "INTERNAL",
|
||
r["candidate_status"],
|
||
)
|
||
for r in candidate_rows
|
||
)
|
||
|
||
report["summary"] = {
|
||
"area_count": area_count,
|
||
"coverage_count": coverage_count,
|
||
|
||
"valid_polygon_records": len(valid_records),
|
||
"invalid_polygon_records": len(invalid_records),
|
||
|
||
"valid_policy_changed_count":
|
||
valid_policy_changed,
|
||
"valid_policy_hash_changed_count":
|
||
valid_policy_hash_changed,
|
||
|
||
"candidate_status_counts":
|
||
dict(status_counts),
|
||
|
||
"candidate_pass_count":
|
||
status_counts.get(
|
||
"PASS_NORMALIZATION_CANDIDATE", 0
|
||
),
|
||
|
||
"candidate_check_count":
|
||
status_counts.get(
|
||
"CHECK_NORMALIZATION_CANDIDATE", 0
|
||
)
|
||
+ status_counts.get(
|
||
"FAIL_NO_CANDIDATE", 0
|
||
),
|
||
|
||
"candidate_valid_count":
|
||
sum(
|
||
int(r.get("candidate_valid") or 0)
|
||
for r in candidate_rows
|
||
),
|
||
|
||
"candidate_polygon_only_count":
|
||
sum(
|
||
int(r.get("candidate_polygon_only") or 0)
|
||
for r in candidate_rows
|
||
),
|
||
|
||
"candidate_matches_make_valid_polygonal_count":
|
||
sum(
|
||
int(
|
||
r.get(
|
||
"candidate_matches_make_valid_polygonal"
|
||
) or 0
|
||
)
|
||
for r in candidate_rows
|
||
),
|
||
|
||
"candidate_no_new_boundary_count":
|
||
sum(
|
||
int(
|
||
r.get(
|
||
"candidate_introduces_no_new_boundary"
|
||
) or 0
|
||
)
|
||
for r in candidate_rows
|
||
),
|
||
|
||
"discarded_matches_make_valid_remainder_count":
|
||
sum(
|
||
int(
|
||
r.get(
|
||
"discarded_matches_make_valid_remainder"
|
||
) or 0
|
||
)
|
||
for r in candidate_rows
|
||
),
|
||
|
||
"idempotent_count":
|
||
sum(
|
||
int(
|
||
r.get(
|
||
"normalization_idempotent"
|
||
) or 0
|
||
)
|
||
for r in candidate_rows
|
||
),
|
||
|
||
"bbox_changed_count":
|
||
sum(
|
||
int(
|
||
r.get(
|
||
"bbox_changed_over_tolerance"
|
||
) or 0
|
||
)
|
||
for r in candidate_rows
|
||
),
|
||
}
|
||
|
||
full_pass = (
|
||
len(invalid_records) == 55
|
||
and report["summary"]["candidate_pass_count"] == 55
|
||
and report["summary"]["candidate_check_count"] == 0
|
||
and valid_policy_changed == 0
|
||
and valid_policy_hash_changed == 0
|
||
)
|
||
|
||
report["summary"]["overall_dryrun_pass"] = full_pass
|
||
|
||
# ----------------------------------------------------------------
|
||
# Outputs
|
||
# ----------------------------------------------------------------
|
||
|
||
print("[5/6] 写 CSV / JSON / GeoJSON ...")
|
||
|
||
write_csv(
|
||
outdir / "normalization_candidate_results.csv",
|
||
candidate_rows
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "normalization_face_selection.csv",
|
||
face_rows
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "normalization_boundary_compare.csv",
|
||
boundary_rows
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "normalization_failure_detail.csv",
|
||
failure_rows
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "normalization_by_object_class.csv",
|
||
[
|
||
{
|
||
"object_class": obj,
|
||
"candidate_status": status,
|
||
"count": count,
|
||
}
|
||
for (obj, status), count
|
||
in object_counts.most_common()
|
||
]
|
||
)
|
||
|
||
write_csv(
|
||
outdir / "normalization_by_boundary_context.csv",
|
||
[
|
||
{
|
||
"boundary_context": ctx,
|
||
"candidate_status": status,
|
||
"count": count,
|
||
}
|
||
for (ctx, status), count
|
||
in boundary_context_counts.most_common()
|
||
]
|
||
)
|
||
|
||
make_geojson(
|
||
records,
|
||
outdir / "normalization_candidates.geojson"
|
||
)
|
||
|
||
(outdir / "normalization_dryrun_report_v9.json").write_text(
|
||
json.dumps(
|
||
report,
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
default=str
|
||
),
|
||
encoding="utf-8"
|
||
)
|
||
|
||
lines = []
|
||
|
||
lines.append(
|
||
"CM93 -> MySQL OGC Polygon Normalization Dry-Run v9"
|
||
)
|
||
lines.append("=" * 92)
|
||
lines.append(
|
||
f"Generated: {report['generated_at']}"
|
||
)
|
||
lines.append(
|
||
f"Dataset ID: {args.dataset_id}"
|
||
)
|
||
lines.append(
|
||
f"MySQL: {report['mysql_version']}"
|
||
)
|
||
lines.append(
|
||
f"Shapely: {report['shapely_version']}"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("SCOPE")
|
||
lines.append(
|
||
" CM93JAPAN 是日本沿海局部 CM93 数据集；"
|
||
"跨未导入海域边界不连续不作为错误。"
|
||
)
|
||
lines.append(
|
||
" V9 只在内存构造 odd-even OGC normalization candidate，"
|
||
"不修改数据库。"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("BASELINE")
|
||
lines.append(
|
||
f" Area records: {area_count}"
|
||
)
|
||
lines.append(
|
||
f" Coverage records: {coverage_count}"
|
||
)
|
||
lines.append(
|
||
f" valid polygon records: {len(valid_records)}"
|
||
)
|
||
lines.append(
|
||
f" invalid polygon records: {len(invalid_records)}"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("VALID PASS-THROUGH GUARD")
|
||
lines.append(
|
||
f" policy changed valid geometry: "
|
||
f"{valid_policy_changed}"
|
||
)
|
||
lines.append(
|
||
f" WKB hash changed valid geometry:"
|
||
f" {valid_policy_hash_changed}"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("CANDIDATE STATUS")
|
||
for k, v in status_counts.most_common():
|
||
lines.append(
|
||
f" {k:44s} {v}"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("PASS GATES")
|
||
lines.append(
|
||
f" candidate valid: "
|
||
f"{report['summary']['candidate_valid_count']}"
|
||
f" / {len(invalid_records)}"
|
||
)
|
||
lines.append(
|
||
f" polygonal-only: "
|
||
f"{report['summary']['candidate_polygon_only_count']}"
|
||
f" / {len(invalid_records)}"
|
||
)
|
||
lines.append(
|
||
f" matches make_valid polygonal: "
|
||
f"{report['summary']['candidate_matches_make_valid_polygonal_count']}"
|
||
f" / {len(invalid_records)}"
|
||
)
|
||
lines.append(
|
||
f" introduces no new boundary: "
|
||
f"{report['summary']['candidate_no_new_boundary_count']}"
|
||
f" / {len(invalid_records)}"
|
||
)
|
||
lines.append(
|
||
f" discarded linework matches remainder: "
|
||
f"{report['summary']['discarded_matches_make_valid_remainder_count']}"
|
||
f" / {len(invalid_records)}"
|
||
)
|
||
lines.append(
|
||
f" normalization idempotent: "
|
||
f"{report['summary']['idempotent_count']}"
|
||
f" / {len(invalid_records)}"
|
||
)
|
||
lines.append(
|
||
f" bbox changed over tolerance: "
|
||
f"{report['summary']['bbox_changed_count']}"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("OVERALL")
|
||
lines.append(
|
||
f" DRY-RUN PASS: "
|
||
f"{'YES' if full_pass else 'NO'}"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append("INTERPRETATION")
|
||
lines.append(
|
||
" 若 55/55 全部 PASS，说明 odd-even face selection 可以稳定生成"
|
||
" OGC-valid Polygon/MultiPolygon candidate，且不会给已有有效 geometry "
|
||
"引入变化。"
|
||
)
|
||
lines.append(
|
||
" discarded linework 只是已确认的 retraced connector 时，"
|
||
"可把它理解为零面积 CM93 topology bridge，而不是 Area 面积。"
|
||
)
|
||
lines.append(
|
||
" 即使 DRY-RUN PASS，本阶段仍不允许写回 Dataset ID=1。"
|
||
)
|
||
lines.append("")
|
||
|
||
lines.append(
|
||
"NO DATABASE WRITE / NO IMPORTER CHANGE / NO PBF"
|
||
)
|
||
|
||
(
|
||
outdir / "normalization_dryrun_summary_v9.txt"
|
||
).write_text(
|
||
"\n".join(lines),
|
||
encoding="utf-8"
|
||
)
|
||
|
||
conn.rollback()
|
||
|
||
print("[6/6] 压缩输出 ...")
|
||
|
||
zip_path = outdir.with_suffix(".zip")
|
||
|
||
with zipfile.ZipFile(
|
||
zip_path,
|
||
"w",
|
||
compression=zipfile.ZIP_DEFLATED
|
||
) as z:
|
||
for p in sorted(outdir.rglob("*")):
|
||
if p.is_file():
|
||
z.write(
|
||
p,
|
||
p.relative_to(outdir.parent)
|
||
)
|
||
|
||
print("")
|
||
print("完成。")
|
||
print(f"Output: {outdir.resolve()}")
|
||
print(f"ZIP: {zip_path.resolve()}")
|
||
print("把 ZIP 整包上传给 ChatGPT。")
|
||
|
||
except KeyboardInterrupt:
|
||
if conn:
|
||
try:
|
||
conn.rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
print("\n用户中断。")
|
||
sys.exit(130)
|
||
|
||
except Exception:
|
||
if conn:
|
||
try:
|
||
conn.rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
tb = traceback.format_exc()
|
||
print(tb)
|
||
|
||
try:
|
||
(
|
||
outdir / "fatal_error.txt"
|
||
).write_text(
|
||
tb,
|
||
encoding="utf-8"
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
sys.exit(1)
|
||
|
||
finally:
|
||
if conn:
|
||
try:
|
||
conn.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|