Files
CM95/cm93_normalization_implementation_precheck_v11.py
2026-09-02 09:50:12 +08:00

1271 lines
43 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
CM93 -> MySQL Normalization 实施前只读预检
Version: v11.0
目标
====
V11 不再依赖把 V10.1 大包上传给 ChatGPT。
它在用户本机完成两个门控:
Gate A:V10.1 canonical validation 必须 PASS
------------------------------------------------
从以下任一输入读取:
- V10.1 完整输出目录
- V10.1 完整 ZIP
- v10.2 digest ZIP
要求至少能确认:
- overall_pass = true
- invalid_polygon_records = 55
- golden_vector_count = 55
- 55 条 golden 全 PASS
- valid_policy_changed_count = 0
- valid_hash_changed_count = 0
- input variant / face order / repeat / topology failures = 0
若不满足,V11 立即停止,不继续。
Gate B:当前 MySQL 数据必须仍与 V10.1 golden vectors 一致
-----------------------------------------------------------
V11 会重新从数据库读取 52 invalid Area + 3 invalid Coverage,
使用已经冻结的规则:
ODD_EVEN_FACE_SELECTION
+
GEOS_NORMALIZE
重新生成 candidate,然后逐条比较:
- canonical WKB SHA-256
- geometry type
- polygon part count
- candidate bbox
- candidate area
必须与 V10.1 golden vector 一致。
之后做实施前 schema 检查:
- chart_feature
- chart_area_geometry
- chart_coverage
- chart_import_job
检查:
- geometry 字段真实 SQL 类型
- 是否能存 MultiPolygon
- SRID / SRS 约束
- 主键、唯一键、外键、索引
- bbox 字段
- geometry_issue / geometry_repaired 字段
- import job geometry counters
- 4406 raw_geotype=16 当前状态
- 55 OGC-invalid 当前状态
- 未来若 normalized geometry 替换原 geometry,需要同步修改哪些 metadata
输出:
- implementation_precheck_summary_v11.txt
- implementation_precheck_report_v11.json
- golden_revalidation_v11.csv
- schema_columns_v11.csv
- schema_indexes_v11.csv
- schema_foreign_keys_v11.csv
- show_create_tables_v11.txt
- normalization_metadata_impact_v11.csv
- proposed_implementation_contract_v11.md
- proposed_migration_plan_v11.sql.txt
注意:
proposed_migration_plan_v11.sql.txt 只是文本计划。
V11 绝不会执行这些 SQL。
禁止:
- 不 UPDATE
- 不 INSERT
- 不 DELETE
- 不 ALTER TABLE
- 不修改 importer
- 不写 normalized geometry
- 不修 4406
- 不 PBF
依赖:
pip install pymysql shapely
执行例 1:直接使用 V10.1 输出目录
python cm93_normalization_implementation_precheck_v11.py ^
--v10-input "D:\\React\\cm93-mysql-importer-starter\\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602" ^
--host 127.0.0.1 ^
--port 3306 ^
--user root ^
--database cm93_chart ^
--dataset-id 1
执行例 2:使用 v10.2 digest ZIP
python cm93_normalization_implementation_precheck_v11.py ^
--v10-input "D:\\...\\cm93_v10_1_result_digest_v10_2.zip" ^
--host 127.0.0.1 ^
--port 3306 ^
--user root ^
--database cm93_chart ^
--dataset-id 1
"""
from __future__ import annotations
import argparse
import csv
import getpass
import hashlib
import json
import math
import os
import shutil
import sys
import tempfile
import zipfile
from collections import Counter
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
try:
import pymysql
from pymysql.cursors import DictCursor
except ImportError:
print("缺少 pymysql:pip install pymysql shapely")
sys.exit(2)
try:
import shapely
try:
from shapely import from_wkb as _from_wkb
def load_wkb(data: bytes):
return _from_wkb(data)
except ImportError:
from shapely import wkb as _wkb
def load_wkb(data: bytes):
return _wkb.loads(data)
try:
from shapely import to_wkb as _to_wkb
def dump_wkb(g) -> bytes:
return bytes(_to_wkb(g, hex=False, byte_order=1))
except ImportError:
from shapely import wkb as _wkb2
def dump_wkb(g) -> bytes:
return _wkb2.dumps(g, hex=False, big_endian=False)
try:
from shapely import normalize as _normalize
def canonicalize(g):
return _normalize(g)
except ImportError:
raise ImportError("V11 需要 Shapely 2.x 的 shapely.normalize()")
from shapely.ops import unary_union, polygonize_full
except ImportError as e:
print(f"Shapely 初始化失败:{e}")
print("请执行:pip install -U shapely")
sys.exit(2)
# ---------------------------------------------------------------------------
# General helpers
# ---------------------------------------------------------------------------
def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any:
cur.execute(sql, params)
row = cur.fetchone()
return next(iter(row.values())) if row else None
def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]:
cur.execute(sql, params)
return list(cur.fetchall())
def begin_read_only(cur, warnings: List[str]):
try:
cur.execute("SET SESSION TRANSACTION READ ONLY")
except Exception as e:
warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}")
try:
cur.execute("START TRANSACTION READ ONLY")
except Exception as e:
warnings.append(f"START TRANSACTION READ ONLY failed: {e}")
cur.execute("START TRANSACTION")
def to_bytes(v: Any) -> Optional[bytes]:
if v is None:
return None
if isinstance(v, bytes):
return v
if isinstance(v, bytearray):
return bytes(v)
if isinstance(v, memoryview):
return v.tobytes()
try:
return bytes(v)
except Exception:
return None
def write_csv(
path: Path,
rows: Iterable[Dict[str, Any]],
fieldnames: Optional[List[str]] = None,
):
rows = list(rows)
if fieldnames is None:
seen = set()
fieldnames = []
for row in rows:
for k in row.keys():
if k not in seen:
seen.add(k)
fieldnames.append(k)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as f:
if not fieldnames:
return
w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
w.writeheader()
for row in rows:
out = {}
for k in fieldnames:
v = row.get(k)
if isinstance(v, (dict, list, tuple)):
v = json.dumps(v, ensure_ascii=False, default=str)
out[k] = "" if v is None else v
w.writerow(out)
def intv(v: Any, default: int = 0) -> int:
try:
return int(v)
except Exception:
return default
def boolv(v: Any) -> bool:
if isinstance(v, bool):
return v
if v is None:
return False
return str(v).strip().lower() in {"1", "true", "yes", "y", "pass"}
def floatv(v: Any, default: Optional[float] = None) -> Optional[float]:
try:
x = float(v)
return x if math.isfinite(x) else default
except Exception:
return default
def geometry_hash(g) -> Optional[str]:
if g is None:
return None
try:
return hashlib.sha256(dump_wkb(g)).hexdigest()
except Exception:
return None
def flatten_components(g) -> Dict[str, List[Any]]:
result = {"polygons": [], "lines": [], "points": [], "others": []}
def rec(x):
if x is None:
return
t = x.geom_type
if t == "Polygon":
result["polygons"].append(x)
elif t == "MultiPolygon":
for y in x.geoms:
rec(y)
elif t in ("LineString", "LinearRing"):
result["lines"].append(x)
elif t == "MultiLineString":
for y in x.geoms:
rec(y)
elif t == "Point":
result["points"].append(x)
elif t == "MultiPoint":
for y in x.geoms:
rec(y)
elif hasattr(x, "geoms"):
for y in x.geoms:
rec(y)
else:
result["others"].append(x)
rec(g)
return result
def polygon_part_count(g) -> int:
if g is None:
return 0
if g.geom_type == "Polygon":
return 1
if g.geom_type == "MultiPolygon":
return len(g.geoms)
return len(flatten_components(g)["polygons"])
def probe_wkb_fn(cur) -> str:
for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"):
try:
cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1")
cur.fetchone()
return fn
except Exception:
pass
raise RuntimeError("找不到 geometry -> WKB 函数")
# ---------------------------------------------------------------------------
# V10 input / gate
# ---------------------------------------------------------------------------
def find_file(root: Path, filename: str) -> Optional[Path]:
direct = root / filename
if direct.exists():
return direct
hits = list(root.rglob(filename))
return hits[0] if hits else None
def read_csv(path: Path) -> List[Dict[str, str]]:
with path.open("r", encoding="utf-8-sig", newline="") as f:
return list(csv.DictReader(f))
def open_v10_input(src: Path) -> Tuple[Path, Optional[Path]]:
if src.is_dir():
return src, None
if src.is_file() and src.suffix.lower() == ".zip":
temp = Path(tempfile.mkdtemp(prefix="cm93_v11_v10_input_"))
with zipfile.ZipFile(src, "r") as z:
z.extractall(temp)
return temp, temp
raise ValueError("v10-input 必须是目录或 ZIP")
def load_v10_gate(root: Path) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]:
digest_json = find_file(root, "V10_1_结果摘要_v10_2.json")
report_json = find_file(root, "normalization_canonical_report_v10_1.json")
golden_csv = find_file(root, "normalization_golden_vectors_v10_1.csv")
gate: Dict[str, Any] = {
"source": None,
"overall_pass": False,
"invalid_polygon_records": None,
"valid_policy_changed_count": None,
"valid_hash_changed_count": None,
"golden_vector_count": None,
"golden_pass_count": None,
"golden_fail_count": None,
"face_order_hash_mismatch_count": None,
"canonical_topology_failure_count": None,
"repeat_hash_failure_count": None,
"input_variant_failure_record_count": None,
"errors": [],
}
golden_map: Dict[str, Dict[str, Any]] = {}
if digest_json:
digest = json.loads(digest_json.read_text(encoding="utf-8"))
gate["source"] = "v10.2-digest"
gate["digest_overall"] = digest.get("overall")
if digest.get("overall") == "PASS":
gate["overall_pass"] = True
checks = digest.get("checks", {})
rc = checks.get("report", {})
gc = checks.get("golden", {})
gate["invalid_polygon_records"] = rc.get("invalid_polygon_records")
gate["valid_policy_changed_count"] = rc.get("valid_policy_changed_count")
gate["valid_hash_changed_count"] = rc.get("valid_hash_changed_count")
gate["golden_vector_count"] = rc.get("golden_vector_count")
gate["golden_pass_count"] = (gc.get("status_counts") or {}).get("PASS")
gate["golden_fail_count"] = (gc.get("status_counts") or {}).get("FAIL", 0)
gate["face_order_hash_mismatch_count"] = rc.get("face_order_hash_mismatch_count")
gate["canonical_topology_failure_count"] = rc.get("canonical_topology_failure_count")
gate["repeat_hash_failure_count"] = rc.get("repeat_hash_failure_count")
gate["input_variant_failure_record_count"] = rc.get("input_variant_failure_record_count")
if report_json:
report = json.loads(report_json.read_text(encoding="utf-8"))
s = report.get("summary", {})
gate["source"] = (
gate["source"] + "+v10.1-report"
if gate["source"]
else "v10.1-report"
)
gate["overall_pass"] = boolv(s.get("overall_pass"))
gate["invalid_polygon_records"] = s.get("invalid_polygon_records")
gate["valid_policy_changed_count"] = s.get("valid_policy_changed_count")
gate["valid_hash_changed_count"] = s.get("valid_hash_changed_count")
gate["golden_vector_count"] = s.get("golden_vector_count")
gate["golden_pass_count"] = (s.get("golden_record_status_counts") or {}).get("PASS")
gate["golden_fail_count"] = (s.get("golden_record_status_counts") or {}).get("FAIL", 0)
gate["face_order_hash_mismatch_count"] = s.get("face_order_hash_mismatch_count")
gate["canonical_topology_failure_count"] = s.get("canonical_topology_failure_count")
gate["repeat_hash_failure_count"] = s.get("repeat_hash_failure_count")
gate["input_variant_failure_record_count"] = s.get("input_variant_failure_record_count")
if golden_csv:
rows = read_csv(golden_csv)
for r in rows:
kind = r.get("kind")
if kind == "AREA":
key = f"AREA:{r.get('feature_id')}"
else:
key = f"COVERAGE:{r.get('coverage_geom_id')}"
golden_map[key] = r
statuses = Counter(str(r.get("record_status", "")).strip() for r in rows)
gate["golden_vector_count"] = len(rows)
gate["golden_pass_count"] = statuses.get("PASS", 0)
gate["golden_fail_count"] = statuses.get("FAIL", 0)
pass_conditions = [
gate["overall_pass"] is True,
intv(gate["invalid_polygon_records"]) == 55,
intv(gate["valid_policy_changed_count"]) == 0,
intv(gate["valid_hash_changed_count"]) == 0,
intv(gate["golden_vector_count"]) == 55,
intv(gate["golden_pass_count"]) == 55,
intv(gate["golden_fail_count"]) == 0,
intv(gate["face_order_hash_mismatch_count"]) == 0,
intv(gate["canonical_topology_failure_count"]) == 0,
intv(gate["repeat_hash_failure_count"]) == 0,
intv(gate["input_variant_failure_record_count"]) == 0,
]
gate["pass_conditions"] = pass_conditions
gate["gate_pass"] = all(pass_conditions)
if len(golden_map) != 55:
gate["errors"].append(
f"Golden vector 明细不足:{len(golden_map)} / 55。"
)
gate["gate_pass"] = False
return gate, golden_map
# ---------------------------------------------------------------------------
# Odd-even normalization
# ---------------------------------------------------------------------------
def original_rings(geom) -> List[List[Tuple[float, float]]]:
rings = []
def add_poly(poly):
rings.append([(float(c[0]), float(c[1])) for c in poly.exterior.coords])
for ring in poly.interiors:
rings.append([(float(c[0]), float(c[1])) for c in ring.coords])
if geom.geom_type == "Polygon":
add_poly(geom)
elif geom.geom_type == "MultiPolygon":
for p in geom.geoms:
add_poly(p)
return rings
def point_on_segment(px, py, ax, ay, bx, by, eps=1e-12) -> bool:
vx, vy = bx - ax, by - ay
wx, wy = px - ax, py - ay
c = abs(vx * wy - vy * wx)
if c > eps * max(1.0, abs(vx), abs(vy)):
return False
d = wx * vx + wy * vy
if d < -eps:
return False
vv = vx * vx + vy * vy
if d - vv > eps:
return False
return True
def even_odd_inside(
point: Tuple[float, float],
rings: List[List[Tuple[float, float]]]
) -> bool:
x, y = point
crossings = 0
for ring in rings:
for i in range(len(ring) - 1):
x1, y1 = ring[i]
x2, y2 = ring[i + 1]
if point_on_segment(x, y, x1, y1, x2, y2):
return True
if (y1 > y) != (y2 > y):
xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1)
if xinters > x:
crossings += 1
return crossings % 2 == 1
def normalize_invalid(geom):
noded = unary_union(geom.boundary)
polygons_geom, cuts, dangles, invalid_rings = polygonize_full(noded)
faces = flatten_components(polygons_geom)["polygons"]
rings = original_rings(geom)
selected = []
for face in faces:
rp = face.representative_point()
if even_odd_inside((float(rp.x), float(rp.y)), rings):
selected.append(face)
if not selected:
return None
return canonicalize(unary_union(selected))
# ---------------------------------------------------------------------------
# DB loaders / schema
# ---------------------------------------------------------------------------
def load_invalid_db_records(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]:
out = []
cur.execute(f"""
SELECT
'AREA' AS kind,
f.id AS feature_id,
NULL AS coverage_geom_id,
f.object_class,
f.cell_id,
c.cell_name,
f.source_record_index,
f.coverage_id,
f.raw_geotype,
f.bbox_min_lon,
f.bbox_min_lat,
f.bbox_max_lon,
f.bbox_max_lat,
{wkb_fn}(g.geom) AS geom_wkb
FROM chart_feature f
JOIN chart_area_geometry g ON g.feature_id=f.id
JOIN chart_cell c ON c.id=f.cell_id
WHERE f.dataset_id=%s AND f.geometry_type='AREA'
ORDER BY f.id
""", (dataset_id,))
for r in cur:
try:
geom = load_wkb(to_bytes(r["geom_wkb"]))
if not geom.is_valid:
x = dict(r)
x["_geom"] = geom
out.append(x)
except Exception:
continue
cur.execute(f"""
SELECT
'COVERAGE' AS kind,
NULL AS feature_id,
v.id AS coverage_geom_id,
'COVERAGE' AS object_class,
v.cell_id,
c.cell_name,
v.source_record_index,
NULL AS coverage_id,
NULL AS raw_geotype,
NULL AS bbox_min_lon,
NULL AS bbox_min_lat,
NULL AS bbox_max_lon,
NULL AS bbox_max_lat,
{wkb_fn}(v.geom) AS geom_wkb
FROM chart_coverage v
JOIN chart_cell c ON c.id=v.cell_id
WHERE v.dataset_id=%s
ORDER BY v.id
""", (dataset_id,))
for r in cur:
try:
geom = load_wkb(to_bytes(r["geom_wkb"]))
if not geom.is_valid:
x = dict(r)
x["_geom"] = geom
out.append(x)
except Exception:
continue
return out
def schema_columns(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]:
ph = ",".join(["%s"] * len(tables))
sql = f"""
SELECT
TABLE_NAME,
ORDINAL_POSITION,
COLUMN_NAME,
DATA_TYPE,
COLUMN_TYPE,
IS_NULLABLE,
COLUMN_DEFAULT,
COLUMN_KEY,
EXTRA,
COLUMN_COMMENT
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA=%s
AND TABLE_NAME IN ({ph})
ORDER BY TABLE_NAME, ORDINAL_POSITION
"""
return fetch_all(cur, sql, [database] + list(tables))
def schema_indexes(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]:
ph = ",".join(["%s"] * len(tables))
sql = f"""
SELECT
TABLE_NAME,
INDEX_NAME,
NON_UNIQUE,
SEQ_IN_INDEX,
COLUMN_NAME,
INDEX_TYPE
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA=%s
AND TABLE_NAME IN ({ph})
ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX
"""
return fetch_all(cur, sql, [database] + list(tables))
def schema_foreign_keys(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]:
ph = ",".join(["%s"] * len(tables))
sql = f"""
SELECT
TABLE_NAME,
COLUMN_NAME,
CONSTRAINT_NAME,
REFERENCED_TABLE_NAME,
REFERENCED_COLUMN_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE TABLE_SCHEMA=%s
AND TABLE_NAME IN ({ph})
AND REFERENCED_TABLE_NAME IS NOT NULL
ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION
"""
return fetch_all(cur, sql, [database] + list(tables))
def show_create(cur, tables: Sequence[str]) -> str:
chunks = []
for table in tables:
try:
cur.execute(f"SHOW CREATE TABLE `{table}`")
row = cur.fetchone()
chunks.append(f"-- {table}\n{list(row.values())[-1]};\n")
except Exception as e:
chunks.append(f"-- {table}: ERROR {e}\n")
return "\n".join(chunks)
def get_column_type(columns: List[Dict[str, Any]], table: str, col: str) -> Optional[str]:
for r in columns:
if r["TABLE_NAME"] == table and r["COLUMN_NAME"] == col:
return str(r["COLUMN_TYPE"])
return None
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def parse_args():
p = argparse.ArgumentParser(
description="CM93 normalization implementation precheck v11"
)
p.add_argument("--v10-input", required=True)
p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1"))
p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306")))
p.add_argument("--user", default=os.getenv("MYSQL_USER", "root"))
p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD"))
p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart"))
p.add_argument("--dataset-id", type=int, default=1)
p.add_argument("--bbox-tolerance", type=float, default=5e-6)
p.add_argument("--area-relative-tolerance", type=float, default=1e-9)
p.add_argument("--output", default=None)
return p.parse_args()
def main():
args = parse_args()
v10_src = Path(args.v10_input).expanduser().resolve()
if not v10_src.exists():
print(f"V10 输入不存在:{v10_src}")
sys.exit(2)
print("[1/7] V10.1 PASS gate ...")
v10_root = None
temp_v10 = None
try:
v10_root, temp_v10 = open_v10_input(v10_src)
gate, golden_map = load_v10_gate(v10_root)
print(json.dumps(gate, ensure_ascii=False, indent=2))
if not gate["gate_pass"]:
print("")
print("STOP:V10.1 gate 未通过。")
print("V11 不连接数据库、不继续实施预检。")
sys.exit(3)
print(" V10.1 gate: PASS")
if not args.password:
args.password = getpass.getpass("MySQL password: ")
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
outdir = Path(
args.output
or f"cm93_normalization_implementation_precheck_v11_dataset_{args.dataset_id}_{stamp}"
)
outdir.mkdir(parents=True, exist_ok=True)
report: Dict[str, Any] = {
"version": "v11.0",
"generated_at": datetime.now().isoformat(timespec="seconds"),
"database": args.database,
"dataset_id": args.dataset_id,
"read_only": True,
"pbf": "OUT_OF_SCOPE",
"v10_gate": gate,
"warnings": [],
}
conn = pymysql.connect(
host=args.host,
port=args.port,
user=args.user,
password=args.password,
database=args.database,
charset="utf8mb4",
cursorclass=DictCursor,
autocommit=False,
connect_timeout=20,
read_timeout=600,
write_timeout=30,
)
try:
with conn.cursor() as cur:
begin_read_only(cur, report["warnings"])
report["mysql_version"] = scalar(cur, "SELECT VERSION()")
wkb_fn = probe_wkb_fn(cur)
print("[2/7] 当前 55 条 invalid 与 golden vectors 重新对账 ...")
db_invalid = load_invalid_db_records(cur, args.dataset_id, wkb_fn)
revalidation = []
metadata_impact = []
for i, r in enumerate(db_invalid, start=1):
key = (
f"AREA:{r['feature_id']}"
if r["kind"] == "AREA"
else f"COVERAGE:{r['coverage_geom_id']}"
)
golden = golden_map.get(key)
if golden is None:
row = {
"key": key,
"status": "FAIL_MISSING_GOLDEN",
}
revalidation.append(row)
continue
candidate = normalize_invalid(r["_geom"])
if candidate is None:
revalidation.append({
"key": key,
"status": "FAIL_NO_CANDIDATE",
})
continue
h = geometry_hash(candidate)
expected_hash = golden.get("canonical_wkb_sha256")
type_match = candidate.geom_type == golden.get("candidate_type")
part_match = (
polygon_part_count(candidate)
== intv(golden.get("candidate_polygon_part_count"))
)
expected_area = floatv(golden.get("candidate_area_deg2"))
area = float(candidate.area)
if expected_area is None:
area_match = False
else:
scale = max(abs(area), abs(expected_area), 1e-30)
area_match = (
abs(area - expected_area)
<= args.area_relative_tolerance * scale
)
try:
expected_bbox_raw = golden.get("candidate_bbox")
expected_bbox = (
json.loads(expected_bbox_raw)
if isinstance(expected_bbox_raw, str)
else expected_bbox_raw
)
if isinstance(expected_bbox, str):
# CSV 里 tuple 可能不是合法 JSON
import ast
expected_bbox = ast.literal_eval(expected_bbox)
expected_bbox = tuple(float(v) for v in expected_bbox)
except Exception:
expected_bbox = None
bbox = tuple(float(v) for v in candidate.bounds)
bbox_match = (
expected_bbox is not None
and len(expected_bbox) == 4
and all(
abs(a - b) <= args.bbox_tolerance
for a, b in zip(bbox, expected_bbox)
)
)
status = (
"PASS"
if (
candidate.is_valid
and candidate.geom_type in {"Polygon", "MultiPolygon"}
and h == expected_hash
and type_match
and part_match
and area_match
and bbox_match
)
else "FAIL"
)
revalidation.append({
"key": key,
"kind": r["kind"],
"feature_id": r.get("feature_id"),
"coverage_geom_id": r.get("coverage_geom_id"),
"object_class": r.get("object_class"),
"cell_name": r.get("cell_name"),
"status": status,
"candidate_valid": int(bool(candidate.is_valid)),
"candidate_type": candidate.geom_type,
"expected_type": golden.get("candidate_type"),
"type_match": int(type_match),
"candidate_part_count": polygon_part_count(candidate),
"expected_part_count": intv(golden.get("candidate_polygon_part_count")),
"part_match": int(part_match),
"candidate_hash": h,
"expected_hash": expected_hash,
"hash_match": int(h == expected_hash),
"candidate_area_deg2": area,
"expected_area_deg2": expected_area,
"area_match": int(area_match),
"candidate_bbox": bbox,
"expected_bbox": expected_bbox,
"bbox_match": int(bbox_match),
})
if r["kind"] == "AREA":
old_bbox = (
floatv(r.get("bbox_min_lon")),
floatv(r.get("bbox_min_lat")),
floatv(r.get("bbox_max_lon")),
floatv(r.get("bbox_max_lat")),
)
old_bbox_valid = all(v is not None for v in old_bbox)
feature_bbox_matches = (
old_bbox_valid
and all(
abs(float(a) - float(b)) <= args.bbox_tolerance
for a, b in zip(old_bbox, bbox)
)
)
else:
old_bbox = None
feature_bbox_matches = None
metadata_impact.append({
"key": key,
"kind": r["kind"],
"feature_id": r.get("feature_id"),
"coverage_geom_id": r.get("coverage_geom_id"),
"object_class": r.get("object_class"),
"cell_name": r.get("cell_name"),
"old_feature_bbox": old_bbox,
"candidate_bbox": bbox,
"feature_bbox_matches_candidate":
(
int(feature_bbox_matches)
if feature_bbox_matches is not None
else None
),
"future_bbox_update_needed":
(
int(not feature_bbox_matches)
if feature_bbox_matches is not None
else None
),
})
print(
f" [{i:02d}/{len(db_invalid)}] {key} -> {status}"
)
rv_status = Counter(r.get("status") for r in revalidation)
print("[3/7] Schema / index / FK 只读调查 ...")
tables = [
"chart_feature",
"chart_area_geometry",
"chart_coverage",
"chart_import_job",
]
columns = schema_columns(cur, args.database, tables)
indexes = schema_indexes(cur, args.database, tables)
fks = schema_foreign_keys(cur, args.database, tables)
creates = show_create(cur, tables)
area_geom_type = get_column_type(
columns, "chart_area_geometry", "geom"
)
coverage_geom_type = get_column_type(
columns, "chart_coverage", "geom"
)
print("[4/7] 4406 parent-no-geometry 与 counters 当前状态 ...")
parent_4406 = int(scalar(cur, """
SELECT COUNT(*)
FROM chart_feature
WHERE dataset_id=%s
AND geometry_type='META'
AND raw_geotype=16
""", (args.dataset_id,)) or 0)
parent_issues = int(scalar(cur, """
SELECT COUNT(*)
FROM chart_feature
WHERE dataset_id=%s
AND geometry_type='META'
AND raw_geotype=16
AND geometry_issue IS NOT NULL
""", (args.dataset_id,)) or 0)
total_feature_issues = int(scalar(cur, """
SELECT COUNT(*)
FROM chart_feature
WHERE dataset_id=%s
AND geometry_issue IS NOT NULL
""", (args.dataset_id,)) or 0)
import_invalid_sum = int(scalar(cur, """
SELECT COALESCE(SUM(geometry_invalid_count),0)
FROM chart_import_job
WHERE dataset_id=%s
""", (args.dataset_id,)) or 0)
bbox_update_count = sum(
int(r.get("future_bbox_update_needed") or 0)
for r in metadata_impact
)
schema_can_store_multipolygon = (
area_geom_type is not None
and area_geom_type.lower().startswith("geometry")
)
coverage_can_store_multipolygon = (
coverage_geom_type is not None
and coverage_geom_type.lower().startswith("geometry")
)
report["summary"] = {
"v10_gate_pass": True,
"db_invalid_count": len(db_invalid),
"golden_revalidation_status_counts": dict(rv_status),
"golden_revalidation_pass_count": rv_status.get("PASS", 0),
"golden_revalidation_fail_count":
len(revalidation) - rv_status.get("PASS", 0),
"area_geometry_column_type": area_geom_type,
"coverage_geometry_column_type": coverage_geom_type,
"area_column_can_store_multipolygon":
schema_can_store_multipolygon,
"coverage_column_can_store_multipolygon":
coverage_can_store_multipolygon,
"future_area_bbox_update_needed_count": bbox_update_count,
"raw_geotype16_meta_count": parent_4406,
"raw_geotype16_with_issue_count": parent_issues,
"total_feature_geometry_issue_count": total_feature_issues,
"import_job_geometry_invalid_sum": import_invalid_sum,
}
implementation_ready = (
len(db_invalid) == 55
and rv_status.get("PASS", 0) == 55
and len(revalidation) == 55
and schema_can_store_multipolygon
and coverage_can_store_multipolygon
)
report["summary"]["implementation_precheck_pass"] = (
implementation_ready
)
print("[5/7] 生成实施契约与 SQL 计划(仅文本) ...")
contract = f"""# CM93 OGC Normalization 实施契约 v11
## 当前门控
- V10.1 canonical gate:PASS
- 当前数据库 55 条 golden revalidation:{rv_status.get("PASS", 0)} / 55 PASS
- Area geometry SQL type:`{area_geom_type}`
- Coverage geometry SQL type:`{coverage_geom_type}`
- Area 可直接容纳 MultiPolygon:{schema_can_store_multipolygon}
- Coverage 可直接容纳 MultiPolygon:{coverage_can_store_multipolygon}
- 未来需要同步更新 Area feature bbox:{bbox_update_count}
## 冻结 normalization 规则
仅当 Polygon/Area geometry 在 OGC/GEOS 意义下 invalid 时执行:
1. 读取原始 polygon boundary。
2. `unary_union(boundary)` 完成 noding。
3. `polygonize_full()` 生成 planar faces。
4. 对每个 face 的 representative point 使用原始 ring 的 **odd-even parity**。
5. 只保留 odd-even inside faces。
6. union selected faces。
7. 使用 GEOS/Shapely `normalize()` canonicalize。
8. 最终结果必须是 `Polygon` 或 `MultiPolygon`。
9. 必须 `is_valid == true`。
10. 现有 valid geometry 必须原样 PASS-THROUGH,不做 canonical rewrite。
## retraced connector
已确认的 retraced connector 是零面积拓扑桥接线:
- 不作为 normalized Area 面积保存;
- raw source / raw feature blob 继续保留;
- 不从原始 CM93 记录中删除。
## bbox
如果 normalized Area bbox 与当前 `chart_feature.bbox_*` 不同:
- normalized geometry 保存时必须同步更新 feature bbox;
- 不得继续保留 connector 扩大的旧 bbox 作为 normalized geometry bbox。
## raw_geotype=16
`raw_geotype=16 && geometry_type=META`:
- 不要求 spatial geometry;
- 不设置 `geometry_issue='geometry missing'`;
- 不增加 `geometry_invalid_count`;
- 不制造 EMPTY geometry。
## 原始数据保留
实施 normalization 时不得删除:
- `raw_feature_blob`
- `chart_raw_record`
- raw geotype
- source_record_index
- object attributes
## PBF
本实施契约仍不启动 PBF。
"""
sql_plan = f"""-- CM93 normalization migration plan v11
-- !!! 仅计划文本,V11 没有执行任何 SQL !!!
--
-- 当前 Area geom 类型: {area_geom_type}
-- 当前 Coverage geom 类型: {coverage_geom_type}
-- 未来 Area bbox 更新数量(当前 Dataset 1): {bbox_update_count}
--
-- 实施建议顺序:
--
-- STEP 1
-- 先修改 importer:
-- raw_geotype=16 / META 不再写 geometry missing / invalid counter。
--
-- STEP 2
-- importer 在 Area/Coverage geometry 写入前:
-- valid -> 原样写入
-- invalid -> odd-even normalization -> GEOS normalize -> valid Polygon/MultiPolygon
--
-- STEP 3
-- normalized Area 写入时重新计算:
-- bbox_min_lon
-- bbox_min_lat
-- bbox_max_lon
-- bbox_max_lat
--
-- STEP 4
-- 对现有 Dataset ID={args.dataset_id} 的历史回填应单独执行,
-- 不应与 importer 代码修改混在一次不可回退操作中。
--
-- STEP 5
-- 历史回填前必须备份/快照,并再次用 V10 golden vectors 校验。
--
-- 本文件故意不生成可直接执行的 UPDATE SQL。
-- 原因:当前阶段只冻结规则和 schema 影响,不执行数据修改。
"""
(outdir / "proposed_implementation_contract_v11.md").write_text(
contract, encoding="utf-8"
)
(outdir / "proposed_migration_plan_v11.sql.txt").write_text(
sql_plan, encoding="utf-8"
)
print("[6/7] 写报告 ...")
write_csv(outdir / "golden_revalidation_v11.csv", revalidation)
write_csv(outdir / "schema_columns_v11.csv", columns)
write_csv(outdir / "schema_indexes_v11.csv", indexes)
write_csv(outdir / "schema_foreign_keys_v11.csv", fks)
write_csv(
outdir / "normalization_metadata_impact_v11.csv",
metadata_impact
)
(outdir / "show_create_tables_v11.txt").write_text(
creates, encoding="utf-8"
)
(outdir / "implementation_precheck_report_v11.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2, default=str),
encoding="utf-8"
)
lines = []
lines.append("CM93 Normalization 实施前只读预检 v11")
lines.append("=" * 88)
lines.append(f"Generated: {report['generated_at']}")
lines.append("")
lines.append("V10 GATE")
lines.append(" PASS")
lines.append("")
lines.append("GOLDEN REVALIDATION")
lines.append(
f" DB invalid: {len(db_invalid)}"
)
lines.append(
f" PASS: {rv_status.get('PASS', 0)}"
)
lines.append(
f" FAIL: {len(revalidation) - rv_status.get('PASS', 0)}"
)
lines.append("")
lines.append("SCHEMA")
lines.append(
f" chart_area_geometry.geom: {area_geom_type}"
)
lines.append(
f" chart_coverage.geom: {coverage_geom_type}"
)
lines.append(
f" Area supports MultiPolygon directly: "
f"{schema_can_store_multipolygon}"
)
lines.append(
f" Coverage supports MultiPolygon directly: "
f"{coverage_can_store_multipolygon}"
)
lines.append("")
lines.append("METADATA")
lines.append(
f" future Area bbox updates needed: {bbox_update_count}"
)
lines.append("")
lines.append("GEOTYPE16")
lines.append(
f" META raw_geotype=16: {parent_4406}"
)
lines.append(
f" still has geometry_issue: {parent_issues}"
)
lines.append(
f" total feature issues: {total_feature_issues}"
)
lines.append(
f" import invalid counter sum:{import_invalid_sum}"
)
lines.append("")
lines.append("OVERALL")
lines.append(
f" IMPLEMENTATION PRECHECK PASS: "
f"{'YES' if implementation_ready else 'NO'}"
)
lines.append("")
lines.append(
"NO DATABASE WRITE / NO ALTER TABLE / NO PBF"
)
(outdir / "implementation_precheck_summary_v11.txt").write_text(
"\n".join(lines), encoding="utf-8"
)
conn.rollback()
print("[7/7] 压缩 ...")
zpath = outdir.with_suffix(".zip")
with zipfile.ZipFile(
zpath, "w", compression=zipfile.ZIP_DEFLATED
) as z:
for p in sorted(outdir.rglob("*")):
if p.is_file():
z.write(p, p.relative_to(outdir.parent))
print("")
print(
"IMPLEMENTATION PRECHECK PASS:",
"YES" if report["summary"]["implementation_precheck_pass"] else "NO"
)
print(f"ZIP: {zpath.resolve()}")
print("把这个 V11 ZIP 上传给 ChatGPT。")
finally:
try:
conn.close()
except Exception:
pass
finally:
if temp_v10 and temp_v10.exists():
shutil.rmtree(temp_v10, ignore_errors=True)
if __name__ == "__main__":
main()