#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ CM93 -> MySQL Normalization 实施前只读预检 Version: v11.0 目标 ==== V11 不再依赖把 V10.1 大包上传给 ChatGPT。 它在用户本机完成两个门控： Gate A：V10.1 canonical validation 必须 PASS ------------------------------------------------ 从以下任一输入读取： - V10.1 完整输出目录 - V10.1 完整 ZIP - v10.2 digest ZIP 要求至少能确认： - overall_pass = true - invalid_polygon_records = 55 - golden_vector_count = 55 - 55 条 golden 全 PASS - valid_policy_changed_count = 0 - valid_hash_changed_count = 0 - input variant / face order / repeat / topology failures = 0 若不满足，V11 立即停止，不继续。 Gate B：当前 MySQL 数据必须仍与 V10.1 golden vectors 一致 ----------------------------------------------------------- V11 会重新从数据库读取 52 invalid Area + 3 invalid Coverage， 使用已经冻结的规则： ODD_EVEN_FACE_SELECTION + GEOS_NORMALIZE 重新生成 candidate，然后逐条比较： - canonical WKB SHA-256 - geometry type - polygon part count - candidate bbox - candidate area 必须与 V10.1 golden vector 一致。 之后做实施前 schema 检查： - chart_feature - chart_area_geometry - chart_coverage - chart_import_job 检查： - geometry 字段真实 SQL 类型 - 是否能存 MultiPolygon - SRID / SRS 约束 - 主键、唯一键、外键、索引 - bbox 字段 - geometry_issue / geometry_repaired 字段 - import job geometry counters - 4406 raw_geotype=16 当前状态 - 55 OGC-invalid 当前状态 - 未来若 normalized geometry 替换原 geometry，需要同步修改哪些 metadata 输出： - implementation_precheck_summary_v11.txt - implementation_precheck_report_v11.json - golden_revalidation_v11.csv - schema_columns_v11.csv - schema_indexes_v11.csv - schema_foreign_keys_v11.csv - show_create_tables_v11.txt - normalization_metadata_impact_v11.csv - proposed_implementation_contract_v11.md - proposed_migration_plan_v11.sql.txt 注意： proposed_migration_plan_v11.sql.txt 只是文本计划。 V11 绝不会执行这些 SQL。 禁止： - 不 UPDATE - 不 INSERT - 不 DELETE - 不 ALTER TABLE - 不修改 importer - 不写 normalized geometry - 不修 4406 - 不 PBF 依赖： pip install pymysql shapely 执行例 1：直接使用 V10.1 输出目录 python cm93_normalization_implementation_precheck_v11.py ^ --v10-input "D:\\React\\cm93-mysql-importer-starter\\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602" ^ --host 127.0.0.1 ^ --port 3306 ^ --user root ^ --database cm93_chart ^ --dataset-id 1 执行例 2：使用 v10.2 digest ZIP python cm93_normalization_implementation_precheck_v11.py ^ --v10-input "D:\\...\\cm93_v10_1_result_digest_v10_2.zip" ^ --host 127.0.0.1 ^ --port 3306 ^ --user root ^ --database cm93_chart ^ --dataset-id 1 """ from __future__ import annotations import argparse import csv import getpass import hashlib import json import math import os import shutil import sys import tempfile import zipfile from collections import Counter from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple try: import pymysql from pymysql.cursors import DictCursor except ImportError: print("缺少 pymysql：pip install pymysql shapely") sys.exit(2) try: import shapely try: from shapely import from_wkb as _from_wkb def load_wkb(data: bytes): return _from_wkb(data) except ImportError: from shapely import wkb as _wkb def load_wkb(data: bytes): return _wkb.loads(data) try: from shapely import to_wkb as _to_wkb def dump_wkb(g) -> bytes: return bytes(_to_wkb(g, hex=False, byte_order=1)) except ImportError: from shapely import wkb as _wkb2 def dump_wkb(g) -> bytes: return _wkb2.dumps(g, hex=False, big_endian=False) try: from shapely import normalize as _normalize def canonicalize(g): return _normalize(g) except ImportError: raise ImportError("V11 需要 Shapely 2.x 的 shapely.normalize()") from shapely.ops import unary_union, polygonize_full except ImportError as e: print(f"Shapely 初始化失败：{e}") print("请执行：pip install -U shapely") sys.exit(2) # --------------------------------------------------------------------------- # General helpers # --------------------------------------------------------------------------- def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: cur.execute(sql, params) row = cur.fetchone() return next(iter(row.values())) if row else None def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: cur.execute(sql, params) return list(cur.fetchall()) def begin_read_only(cur, warnings: List[str]): try: cur.execute("SET SESSION TRANSACTION READ ONLY") except Exception as e: warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") try: cur.execute("START TRANSACTION READ ONLY") except Exception as e: warnings.append(f"START TRANSACTION READ ONLY failed: {e}") cur.execute("START TRANSACTION") def to_bytes(v: Any) -> Optional[bytes]: if v is None: return None if isinstance(v, bytes): return v if isinstance(v, bytearray): return bytes(v) if isinstance(v, memoryview): return v.tobytes() try: return bytes(v) except Exception: return None def write_csv( path: Path, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None, ): rows = list(rows) if fieldnames is None: seen = set() fieldnames = [] for row in rows: for k in row.keys(): if k not in seen: seen.add(k) fieldnames.append(k) path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8-sig", newline="") as f: if not fieldnames: return w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") w.writeheader() for row in rows: out = {} for k in fieldnames: v = row.get(k) if isinstance(v, (dict, list, tuple)): v = json.dumps(v, ensure_ascii=False, default=str) out[k] = "" if v is None else v w.writerow(out) def intv(v: Any, default: int = 0) -> int: try: return int(v) except Exception: return default def boolv(v: Any) -> bool: if isinstance(v, bool): return v if v is None: return False return str(v).strip().lower() in {"1", "true", "yes", "y", "pass"} def floatv(v: Any, default: Optional[float] = None) -> Optional[float]: try: x = float(v) return x if math.isfinite(x) else default except Exception: return default def geometry_hash(g) -> Optional[str]: if g is None: return None try: return hashlib.sha256(dump_wkb(g)).hexdigest() except Exception: return None def flatten_components(g) -> Dict[str, List[Any]]: result = {"polygons": [], "lines": [], "points": [], "others": []} def rec(x): if x is None: return t = x.geom_type if t == "Polygon": result["polygons"].append(x) elif t == "MultiPolygon": for y in x.geoms: rec(y) elif t in ("LineString", "LinearRing"): result["lines"].append(x) elif t == "MultiLineString": for y in x.geoms: rec(y) elif t == "Point": result["points"].append(x) elif t == "MultiPoint": for y in x.geoms: rec(y) elif hasattr(x, "geoms"): for y in x.geoms: rec(y) else: result["others"].append(x) rec(g) return result def polygon_part_count(g) -> int: if g is None: return 0 if g.geom_type == "Polygon": return 1 if g.geom_type == "MultiPolygon": return len(g.geoms) return len(flatten_components(g)["polygons"]) def probe_wkb_fn(cur) -> str: for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): try: cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1") cur.fetchone() return fn except Exception: pass raise RuntimeError("找不到 geometry -> WKB 函数") # --------------------------------------------------------------------------- # V10 input / gate # --------------------------------------------------------------------------- def find_file(root: Path, filename: str) -> Optional[Path]: direct = root / filename if direct.exists(): return direct hits = list(root.rglob(filename)) return hits[0] if hits else None def read_csv(path: Path) -> List[Dict[str, str]]: with path.open("r", encoding="utf-8-sig", newline="") as f: return list(csv.DictReader(f)) def open_v10_input(src: Path) -> Tuple[Path, Optional[Path]]: if src.is_dir(): return src, None if src.is_file() and src.suffix.lower() == ".zip": temp = Path(tempfile.mkdtemp(prefix="cm93_v11_v10_input_")) with zipfile.ZipFile(src, "r") as z: z.extractall(temp) return temp, temp raise ValueError("v10-input 必须是目录或 ZIP") def load_v10_gate(root: Path) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]: digest_json = find_file(root, "V10_1_结果摘要_v10_2.json") report_json = find_file(root, "normalization_canonical_report_v10_1.json") golden_csv = find_file(root, "normalization_golden_vectors_v10_1.csv") gate: Dict[str, Any] = { "source": None, "overall_pass": False, "invalid_polygon_records": None, "valid_policy_changed_count": None, "valid_hash_changed_count": None, "golden_vector_count": None, "golden_pass_count": None, "golden_fail_count": None, "face_order_hash_mismatch_count": None, "canonical_topology_failure_count": None, "repeat_hash_failure_count": None, "input_variant_failure_record_count": None, "errors": [], } golden_map: Dict[str, Dict[str, Any]] = {} if digest_json: digest = json.loads(digest_json.read_text(encoding="utf-8")) gate["source"] = "v10.2-digest" gate["digest_overall"] = digest.get("overall") if digest.get("overall") == "PASS": gate["overall_pass"] = True checks = digest.get("checks", {}) rc = checks.get("report", {}) gc = checks.get("golden", {}) gate["invalid_polygon_records"] = rc.get("invalid_polygon_records") gate["valid_policy_changed_count"] = rc.get("valid_policy_changed_count") gate["valid_hash_changed_count"] = rc.get("valid_hash_changed_count") gate["golden_vector_count"] = rc.get("golden_vector_count") gate["golden_pass_count"] = (gc.get("status_counts") or {}).get("PASS") gate["golden_fail_count"] = (gc.get("status_counts") or {}).get("FAIL", 0) gate["face_order_hash_mismatch_count"] = rc.get("face_order_hash_mismatch_count") gate["canonical_topology_failure_count"] = rc.get("canonical_topology_failure_count") gate["repeat_hash_failure_count"] = rc.get("repeat_hash_failure_count") gate["input_variant_failure_record_count"] = rc.get("input_variant_failure_record_count") if report_json: report = json.loads(report_json.read_text(encoding="utf-8")) s = report.get("summary", {}) gate["source"] = ( gate["source"] + "+v10.1-report" if gate["source"] else "v10.1-report" ) gate["overall_pass"] = boolv(s.get("overall_pass")) gate["invalid_polygon_records"] = s.get("invalid_polygon_records") gate["valid_policy_changed_count"] = s.get("valid_policy_changed_count") gate["valid_hash_changed_count"] = s.get("valid_hash_changed_count") gate["golden_vector_count"] = s.get("golden_vector_count") gate["golden_pass_count"] = (s.get("golden_record_status_counts") or {}).get("PASS") gate["golden_fail_count"] = (s.get("golden_record_status_counts") or {}).get("FAIL", 0) gate["face_order_hash_mismatch_count"] = s.get("face_order_hash_mismatch_count") gate["canonical_topology_failure_count"] = s.get("canonical_topology_failure_count") gate["repeat_hash_failure_count"] = s.get("repeat_hash_failure_count") gate["input_variant_failure_record_count"] = s.get("input_variant_failure_record_count") if golden_csv: rows = read_csv(golden_csv) for r in rows: kind = r.get("kind") if kind == "AREA": key = f"AREA:{r.get('feature_id')}" else: key = f"COVERAGE:{r.get('coverage_geom_id')}" golden_map[key] = r statuses = Counter(str(r.get("record_status", "")).strip() for r in rows) gate["golden_vector_count"] = len(rows) gate["golden_pass_count"] = statuses.get("PASS", 0) gate["golden_fail_count"] = statuses.get("FAIL", 0) pass_conditions = [ gate["overall_pass"] is True, intv(gate["invalid_polygon_records"]) == 55, intv(gate["valid_policy_changed_count"]) == 0, intv(gate["valid_hash_changed_count"]) == 0, intv(gate["golden_vector_count"]) == 55, intv(gate["golden_pass_count"]) == 55, intv(gate["golden_fail_count"]) == 0, intv(gate["face_order_hash_mismatch_count"]) == 0, intv(gate["canonical_topology_failure_count"]) == 0, intv(gate["repeat_hash_failure_count"]) == 0, intv(gate["input_variant_failure_record_count"]) == 0, ] gate["pass_conditions"] = pass_conditions gate["gate_pass"] = all(pass_conditions) if len(golden_map) != 55: gate["errors"].append( f"Golden vector 明细不足：{len(golden_map)} / 55。" ) gate["gate_pass"] = False return gate, golden_map # --------------------------------------------------------------------------- # Odd-even normalization # --------------------------------------------------------------------------- def original_rings(geom) -> List[List[Tuple[float, float]]]: rings = [] def add_poly(poly): rings.append([(float(c[0]), float(c[1])) for c in poly.exterior.coords]) for ring in poly.interiors: rings.append([(float(c[0]), float(c[1])) for c in ring.coords]) if geom.geom_type == "Polygon": add_poly(geom) elif geom.geom_type == "MultiPolygon": for p in geom.geoms: add_poly(p) return rings def point_on_segment(px, py, ax, ay, bx, by, eps=1e-12) -> bool: vx, vy = bx - ax, by - ay wx, wy = px - ax, py - ay c = abs(vx * wy - vy * wx) if c > eps * max(1.0, abs(vx), abs(vy)): return False d = wx * vx + wy * vy if d < -eps: return False vv = vx * vx + vy * vy if d - vv > eps: return False return True def even_odd_inside( point: Tuple[float, float], rings: List[List[Tuple[float, float]]] ) -> bool: x, y = point crossings = 0 for ring in rings: for i in range(len(ring) - 1): x1, y1 = ring[i] x2, y2 = ring[i + 1] if point_on_segment(x, y, x1, y1, x2, y2): return True if (y1 > y) != (y2 > y): xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1) if xinters > x: crossings += 1 return crossings % 2 == 1 def normalize_invalid(geom): noded = unary_union(geom.boundary) polygons_geom, cuts, dangles, invalid_rings = polygonize_full(noded) faces = flatten_components(polygons_geom)["polygons"] rings = original_rings(geom) selected = [] for face in faces: rp = face.representative_point() if even_odd_inside((float(rp.x), float(rp.y)), rings): selected.append(face) if not selected: return None return canonicalize(unary_union(selected)) # --------------------------------------------------------------------------- # DB loaders / schema # --------------------------------------------------------------------------- def load_invalid_db_records(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: out = [] cur.execute(f""" SELECT 'AREA' AS kind, f.id AS feature_id, NULL AS coverage_geom_id, f.object_class, f.cell_id, c.cell_name, f.source_record_index, f.coverage_id, f.raw_geotype, f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, {wkb_fn}(g.geom) AS geom_wkb FROM chart_feature f JOIN chart_area_geometry g ON g.feature_id=f.id JOIN chart_cell c ON c.id=f.cell_id WHERE f.dataset_id=%s AND f.geometry_type='AREA' ORDER BY f.id """, (dataset_id,)) for r in cur: try: geom = load_wkb(to_bytes(r["geom_wkb"])) if not geom.is_valid: x = dict(r) x["_geom"] = geom out.append(x) except Exception: continue cur.execute(f""" SELECT 'COVERAGE' AS kind, NULL AS feature_id, v.id AS coverage_geom_id, 'COVERAGE' AS object_class, v.cell_id, c.cell_name, v.source_record_index, NULL AS coverage_id, NULL AS raw_geotype, NULL AS bbox_min_lon, NULL AS bbox_min_lat, NULL AS bbox_max_lon, NULL AS bbox_max_lat, {wkb_fn}(v.geom) AS geom_wkb FROM chart_coverage v JOIN chart_cell c ON c.id=v.cell_id WHERE v.dataset_id=%s ORDER BY v.id """, (dataset_id,)) for r in cur: try: geom = load_wkb(to_bytes(r["geom_wkb"])) if not geom.is_valid: x = dict(r) x["_geom"] = geom out.append(x) except Exception: continue return out def schema_columns(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]: ph = ",".join(["%s"] * len(tables)) sql = f""" SELECT TABLE_NAME, ORDINAL_POSITION, COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, COLUMN_KEY, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=%s AND TABLE_NAME IN ({ph}) ORDER BY TABLE_NAME, ORDINAL_POSITION """ return fetch_all(cur, sql, [database] + list(tables)) def schema_indexes(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]: ph = ",".join(["%s"] * len(tables)) sql = f""" SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=%s AND TABLE_NAME IN ({ph}) ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX """ return fetch_all(cur, sql, [database] + list(tables)) def schema_foreign_keys(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]: ph = ",".join(["%s"] * len(tables)) sql = f""" SELECT TABLE_NAME, COLUMN_NAME, CONSTRAINT_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=%s AND TABLE_NAME IN ({ph}) AND REFERENCED_TABLE_NAME IS NOT NULL ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION """ return fetch_all(cur, sql, [database] + list(tables)) def show_create(cur, tables: Sequence[str]) -> str: chunks = [] for table in tables: try: cur.execute(f"SHOW CREATE TABLE `{table}`") row = cur.fetchone() chunks.append(f"-- {table}\n{list(row.values())[-1]};\n") except Exception as e: chunks.append(f"-- {table}: ERROR {e}\n") return "\n".join(chunks) def get_column_type(columns: List[Dict[str, Any]], table: str, col: str) -> Optional[str]: for r in columns: if r["TABLE_NAME"] == table and r["COLUMN_NAME"] == col: return str(r["COLUMN_TYPE"]) return None # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def parse_args(): p = argparse.ArgumentParser( description="CM93 normalization implementation precheck v11" ) p.add_argument("--v10-input", required=True) p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) p.add_argument("--dataset-id", type=int, default=1) p.add_argument("--bbox-tolerance", type=float, default=5e-6) p.add_argument("--area-relative-tolerance", type=float, default=1e-9) p.add_argument("--output", default=None) return p.parse_args() def main(): args = parse_args() v10_src = Path(args.v10_input).expanduser().resolve() if not v10_src.exists(): print(f"V10 输入不存在：{v10_src}") sys.exit(2) print("[1/7] V10.1 PASS gate ...") v10_root = None temp_v10 = None try: v10_root, temp_v10 = open_v10_input(v10_src) gate, golden_map = load_v10_gate(v10_root) print(json.dumps(gate, ensure_ascii=False, indent=2)) if not gate["gate_pass"]: print("") print("STOP：V10.1 gate 未通过。") print("V11 不连接数据库、不继续实施预检。") sys.exit(3) print(" V10.1 gate: PASS") if not args.password: args.password = getpass.getpass("MySQL password: ") stamp = datetime.now().strftime("%Y%m%d_%H%M%S") outdir = Path( args.output or f"cm93_normalization_implementation_precheck_v11_dataset_{args.dataset_id}_{stamp}" ) outdir.mkdir(parents=True, exist_ok=True) report: Dict[str, Any] = { "version": "v11.0", "generated_at": datetime.now().isoformat(timespec="seconds"), "database": args.database, "dataset_id": args.dataset_id, "read_only": True, "pbf": "OUT_OF_SCOPE", "v10_gate": gate, "warnings": [], } conn = pymysql.connect( host=args.host, port=args.port, user=args.user, password=args.password, database=args.database, charset="utf8mb4", cursorclass=DictCursor, autocommit=False, connect_timeout=20, read_timeout=600, write_timeout=30, ) try: with conn.cursor() as cur: begin_read_only(cur, report["warnings"]) report["mysql_version"] = scalar(cur, "SELECT VERSION()") wkb_fn = probe_wkb_fn(cur) print("[2/7] 当前 55 条 invalid 与 golden vectors 重新对账 ...") db_invalid = load_invalid_db_records(cur, args.dataset_id, wkb_fn) revalidation = [] metadata_impact = [] for i, r in enumerate(db_invalid, start=1): key = ( f"AREA:{r['feature_id']}" if r["kind"] == "AREA" else f"COVERAGE:{r['coverage_geom_id']}" ) golden = golden_map.get(key) if golden is None: row = { "key": key, "status": "FAIL_MISSING_GOLDEN", } revalidation.append(row) continue candidate = normalize_invalid(r["_geom"]) if candidate is None: revalidation.append({ "key": key, "status": "FAIL_NO_CANDIDATE", }) continue h = geometry_hash(candidate) expected_hash = golden.get("canonical_wkb_sha256") type_match = candidate.geom_type == golden.get("candidate_type") part_match = ( polygon_part_count(candidate) == intv(golden.get("candidate_polygon_part_count")) ) expected_area = floatv(golden.get("candidate_area_deg2")) area = float(candidate.area) if expected_area is None: area_match = False else: scale = max(abs(area), abs(expected_area), 1e-30) area_match = ( abs(area - expected_area) <= args.area_relative_tolerance * scale ) try: expected_bbox_raw = golden.get("candidate_bbox") expected_bbox = ( json.loads(expected_bbox_raw) if isinstance(expected_bbox_raw, str) else expected_bbox_raw ) if isinstance(expected_bbox, str): # CSV 里 tuple 可能不是合法 JSON import ast expected_bbox = ast.literal_eval(expected_bbox) expected_bbox = tuple(float(v) for v in expected_bbox) except Exception: expected_bbox = None bbox = tuple(float(v) for v in candidate.bounds) bbox_match = ( expected_bbox is not None and len(expected_bbox) == 4 and all( abs(a - b) <= args.bbox_tolerance for a, b in zip(bbox, expected_bbox) ) ) status = ( "PASS" if ( candidate.is_valid and candidate.geom_type in {"Polygon", "MultiPolygon"} and h == expected_hash and type_match and part_match and area_match and bbox_match ) else "FAIL" ) revalidation.append({ "key": key, "kind": r["kind"], "feature_id": r.get("feature_id"), "coverage_geom_id": r.get("coverage_geom_id"), "object_class": r.get("object_class"), "cell_name": r.get("cell_name"), "status": status, "candidate_valid": int(bool(candidate.is_valid)), "candidate_type": candidate.geom_type, "expected_type": golden.get("candidate_type"), "type_match": int(type_match), "candidate_part_count": polygon_part_count(candidate), "expected_part_count": intv(golden.get("candidate_polygon_part_count")), "part_match": int(part_match), "candidate_hash": h, "expected_hash": expected_hash, "hash_match": int(h == expected_hash), "candidate_area_deg2": area, "expected_area_deg2": expected_area, "area_match": int(area_match), "candidate_bbox": bbox, "expected_bbox": expected_bbox, "bbox_match": int(bbox_match), }) if r["kind"] == "AREA": old_bbox = ( floatv(r.get("bbox_min_lon")), floatv(r.get("bbox_min_lat")), floatv(r.get("bbox_max_lon")), floatv(r.get("bbox_max_lat")), ) old_bbox_valid = all(v is not None for v in old_bbox) feature_bbox_matches = ( old_bbox_valid and all( abs(float(a) - float(b)) <= args.bbox_tolerance for a, b in zip(old_bbox, bbox) ) ) else: old_bbox = None feature_bbox_matches = None metadata_impact.append({ "key": key, "kind": r["kind"], "feature_id": r.get("feature_id"), "coverage_geom_id": r.get("coverage_geom_id"), "object_class": r.get("object_class"), "cell_name": r.get("cell_name"), "old_feature_bbox": old_bbox, "candidate_bbox": bbox, "feature_bbox_matches_candidate": ( int(feature_bbox_matches) if feature_bbox_matches is not None else None ), "future_bbox_update_needed": ( int(not feature_bbox_matches) if feature_bbox_matches is not None else None ), }) print( f" [{i:02d}/{len(db_invalid)}] {key} -> {status}" ) rv_status = Counter(r.get("status") for r in revalidation) print("[3/7] Schema / index / FK 只读调查 ...") tables = [ "chart_feature", "chart_area_geometry", "chart_coverage", "chart_import_job", ] columns = schema_columns(cur, args.database, tables) indexes = schema_indexes(cur, args.database, tables) fks = schema_foreign_keys(cur, args.database, tables) creates = show_create(cur, tables) area_geom_type = get_column_type( columns, "chart_area_geometry", "geom" ) coverage_geom_type = get_column_type( columns, "chart_coverage", "geom" ) print("[4/7] 4406 parent-no-geometry 与 counters 当前状态 ...") parent_4406 = int(scalar(cur, """ SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_type='META' AND raw_geotype=16 """, (args.dataset_id,)) or 0) parent_issues = int(scalar(cur, """ SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_type='META' AND raw_geotype=16 AND geometry_issue IS NOT NULL """, (args.dataset_id,)) or 0) total_feature_issues = int(scalar(cur, """ SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_issue IS NOT NULL """, (args.dataset_id,)) or 0) import_invalid_sum = int(scalar(cur, """ SELECT COALESCE(SUM(geometry_invalid_count),0) FROM chart_import_job WHERE dataset_id=%s """, (args.dataset_id,)) or 0) bbox_update_count = sum( int(r.get("future_bbox_update_needed") or 0) for r in metadata_impact ) schema_can_store_multipolygon = ( area_geom_type is not None and area_geom_type.lower().startswith("geometry") ) coverage_can_store_multipolygon = ( coverage_geom_type is not None and coverage_geom_type.lower().startswith("geometry") ) report["summary"] = { "v10_gate_pass": True, "db_invalid_count": len(db_invalid), "golden_revalidation_status_counts": dict(rv_status), "golden_revalidation_pass_count": rv_status.get("PASS", 0), "golden_revalidation_fail_count": len(revalidation) - rv_status.get("PASS", 0), "area_geometry_column_type": area_geom_type, "coverage_geometry_column_type": coverage_geom_type, "area_column_can_store_multipolygon": schema_can_store_multipolygon, "coverage_column_can_store_multipolygon": coverage_can_store_multipolygon, "future_area_bbox_update_needed_count": bbox_update_count, "raw_geotype16_meta_count": parent_4406, "raw_geotype16_with_issue_count": parent_issues, "total_feature_geometry_issue_count": total_feature_issues, "import_job_geometry_invalid_sum": import_invalid_sum, } implementation_ready = ( len(db_invalid) == 55 and rv_status.get("PASS", 0) == 55 and len(revalidation) == 55 and schema_can_store_multipolygon and coverage_can_store_multipolygon ) report["summary"]["implementation_precheck_pass"] = ( implementation_ready ) print("[5/7] 生成实施契约与 SQL 计划（仅文本） ...") contract = f"""# CM93 OGC Normalization 实施契约 v11 ## 当前门控 - V10.1 canonical gate：PASS - 当前数据库 55 条 golden revalidation：{rv_status.get("PASS", 0)} / 55 PASS - Area geometry SQL type：`{area_geom_type}` - Coverage geometry SQL type：`{coverage_geom_type}` - Area 可直接容纳 MultiPolygon：{schema_can_store_multipolygon} - Coverage 可直接容纳 MultiPolygon：{coverage_can_store_multipolygon} - 未来需要同步更新 Area feature bbox：{bbox_update_count} ## 冻结 normalization 规则 仅当 Polygon/Area geometry 在 OGC/GEOS 意义下 invalid 时执行： 1. 读取原始 polygon boundary。 2. `unary_union(boundary)` 完成 noding。 3. `polygonize_full()` 生成 planar faces。 4. 对每个 face 的 representative point 使用原始 ring 的 **odd-even parity**。 5. 只保留 odd-even inside faces。 6. union selected faces。 7. 使用 GEOS/Shapely `normalize()` canonicalize。 8. 最终结果必须是 `Polygon` 或 `MultiPolygon`。 9. 必须 `is_valid == true`。 10. 现有 valid geometry 必须原样 PASS-THROUGH，不做 canonical rewrite。 ## retraced connector 已确认的 retraced connector 是零面积拓扑桥接线： - 不作为 normalized Area 面积保存； - raw source / raw feature blob 继续保留； - 不从原始 CM93 记录中删除。 ## bbox 如果 normalized Area bbox 与当前 `chart_feature.bbox_*` 不同： - normalized geometry 保存时必须同步更新 feature bbox； - 不得继续保留 connector 扩大的旧 bbox 作为 normalized geometry bbox。 ## raw_geotype=16 `raw_geotype=16 && geometry_type=META`： - 不要求 spatial geometry； - 不设置 `geometry_issue='geometry missing'`； - 不增加 `geometry_invalid_count`； - 不制造 EMPTY geometry。 ## 原始数据保留 实施 normalization 时不得删除： - `raw_feature_blob` - `chart_raw_record` - raw geotype - source_record_index - object attributes ## PBF 本实施契约仍不启动 PBF。 """ sql_plan = f"""-- CM93 normalization migration plan v11 -- !!! 仅计划文本，V11 没有执行任何 SQL !!! -- -- 当前 Area geom 类型: {area_geom_type} -- 当前 Coverage geom 类型: {coverage_geom_type} -- 未来 Area bbox 更新数量（当前 Dataset 1）: {bbox_update_count} -- -- 实施建议顺序： -- -- STEP 1 -- 先修改 importer： -- raw_geotype=16 / META 不再写 geometry missing / invalid counter。 -- -- STEP 2 -- importer 在 Area/Coverage geometry 写入前： -- valid -> 原样写入 -- invalid -> odd-even normalization -> GEOS normalize -> valid Polygon/MultiPolygon -- -- STEP 3 -- normalized Area 写入时重新计算： -- bbox_min_lon -- bbox_min_lat -- bbox_max_lon -- bbox_max_lat -- -- STEP 4 -- 对现有 Dataset ID={args.dataset_id} 的历史回填应单独执行， -- 不应与 importer 代码修改混在一次不可回退操作中。 -- -- STEP 5 -- 历史回填前必须备份/快照，并再次用 V10 golden vectors 校验。 -- -- 本文件故意不生成可直接执行的 UPDATE SQL。 -- 原因：当前阶段只冻结规则和 schema 影响，不执行数据修改。 """ (outdir / "proposed_implementation_contract_v11.md").write_text( contract, encoding="utf-8" ) (outdir / "proposed_migration_plan_v11.sql.txt").write_text( sql_plan, encoding="utf-8" ) print("[6/7] 写报告 ...") write_csv(outdir / "golden_revalidation_v11.csv", revalidation) write_csv(outdir / "schema_columns_v11.csv", columns) write_csv(outdir / "schema_indexes_v11.csv", indexes) write_csv(outdir / "schema_foreign_keys_v11.csv", fks) write_csv( outdir / "normalization_metadata_impact_v11.csv", metadata_impact ) (outdir / "show_create_tables_v11.txt").write_text( creates, encoding="utf-8" ) (outdir / "implementation_precheck_report_v11.json").write_text( json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding="utf-8" ) lines = [] lines.append("CM93 Normalization 实施前只读预检 v11") lines.append("=" * 88) lines.append(f"Generated: {report['generated_at']}") lines.append("") lines.append("V10 GATE") lines.append(" PASS") lines.append("") lines.append("GOLDEN REVALIDATION") lines.append( f" DB invalid: {len(db_invalid)}" ) lines.append( f" PASS: {rv_status.get('PASS', 0)}" ) lines.append( f" FAIL: {len(revalidation) - rv_status.get('PASS', 0)}" ) lines.append("") lines.append("SCHEMA") lines.append( f" chart_area_geometry.geom: {area_geom_type}" ) lines.append( f" chart_coverage.geom: {coverage_geom_type}" ) lines.append( f" Area supports MultiPolygon directly: " f"{schema_can_store_multipolygon}" ) lines.append( f" Coverage supports MultiPolygon directly: " f"{coverage_can_store_multipolygon}" ) lines.append("") lines.append("METADATA") lines.append( f" future Area bbox updates needed: {bbox_update_count}" ) lines.append("") lines.append("GEOTYPE16") lines.append( f" META raw_geotype=16: {parent_4406}" ) lines.append( f" still has geometry_issue: {parent_issues}" ) lines.append( f" total feature issues: {total_feature_issues}" ) lines.append( f" import invalid counter sum:{import_invalid_sum}" ) lines.append("") lines.append("OVERALL") lines.append( f" IMPLEMENTATION PRECHECK PASS: " f"{'YES' if implementation_ready else 'NO'}" ) lines.append("") lines.append( "NO DATABASE WRITE / NO ALTER TABLE / NO PBF" ) (outdir / "implementation_precheck_summary_v11.txt").write_text( "\n".join(lines), encoding="utf-8" ) conn.rollback() print("[7/7] 压缩 ...") zpath = outdir.with_suffix(".zip") with zipfile.ZipFile( zpath, "w", compression=zipfile.ZIP_DEFLATED ) as z: for p in sorted(outdir.rglob("*")): if p.is_file(): z.write(p, p.relative_to(outdir.parent)) print("") print( "IMPLEMENTATION PRECHECK PASS:", "YES" if report["summary"]["implementation_precheck_pass"] else "NO" ) print(f"ZIP: {zpath.resolve()}") print("把这个 V11 ZIP 上传给 ChatGPT。") finally: try: conn.close() except Exception: pass finally: if temp_v10 and temp_v10.exists(): shutil.rmtree(temp_v10, ignore_errors=True) if __name__ == "__main__": main()