Freeze validated CM93 importer baseline docs and audits

tools/cm93_mysql_final_geometry_audit.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+import argparse, csv, getpass, json, sys
+from datetime import datetime
+from pathlib import Path
+
+try:
+    import pymysql
+    from pymysql.cursors import DictCursor
+except ImportError:
+    print("缺少 pymysql：pip install pymysql")
+    sys.exit(2)
+
+try:
+    try:
+        from shapely import from_wkb
+        def load_wkb(x): return from_wkb(x)
+    except ImportError:
+        from shapely import wkb
+        def load_wkb(x): return wkb.loads(x)
+except ImportError:
+    print("缺少 shapely：pip install shapely")
+    sys.exit(2)
+
+def scalar(cur, sql, params=()):
+    cur.execute(sql, params)
+    r = cur.fetchone()
+    return None if not r else next(iter(r.values()))
+
+def as_bytes(v):
+    if isinstance(v, memoryview): return v.tobytes()
+    if isinstance(v, bytearray): return bytes(v)
+    return v
+
+def wkb_fn(cur, table):
+    for fn in ("ST_AsBinary","ST_AsWKB","AsBinary","AsWKB"):
+        try:
+            cur.execute(f"SELECT {fn}(geom) AS g FROM {table} LIMIT 1")
+            cur.fetchone()
+            return fn
+        except Exception:
+            pass
+    raise RuntimeError(f"{table}: 无可用 Geometry->WKB 函数")
+
+def main():
+    ap = argparse.ArgumentParser(description="CM93 normalization 最终全库 Geometry 验收")
+    ap.add_argument("--host", default="127.0.0.1")
+    ap.add_argument("--port", type=int, default=3306)
+    ap.add_argument("--user", default="root")
+    ap.add_argument("--password")
+    ap.add_argument("--database", default="cm93_chart")
+    ap.add_argument("--dataset-id", type=int, default=1)
+    ap.add_argument("--bbox-tolerance", type=float, default=5e-6)
+    a = ap.parse_args()
+    if a.password is None:
+        a.password = getpass.getpass("MySQL password: ")
+
+    stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    outdir = Path(f"cm93_final_geometry_audit_{stamp}")
+    outdir.mkdir(exist_ok=True)
+
+    conn = pymysql.connect(
+        host=a.host, port=a.port, user=a.user, password=a.password,
+        database=a.database, charset="utf8mb4",
+        cursorclass=DictCursor, autocommit=False,
+        connect_timeout=20, read_timeout=900
+    )
+
+    failures = []
+    report = {"dataset_id": a.dataset_id, "generated_at": datetime.now().isoformat()}
+
+    try:
+        with conn.cursor() as cur:
+            try: cur.execute("SET SESSION TRANSACTION READ ONLY")
+            except Exception: pass
+            try: cur.execute("START TRANSACTION READ ONLY")
+            except Exception: cur.execute("START TRANSACTION")
+
+            counts = {
+                "cells": int(scalar(cur,"SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s",(a.dataset_id,)) or 0),
+                "features": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s",(a.dataset_id,)) or 0),
+                "feature_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0),
+                "coverage_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0),
+                "feature_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0),
+                "coverage_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0),
+                "geotype16_total": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16",(a.dataset_id,)) or 0),
+                "geotype16_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16 AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0),
+            }
+            report["counts"] = counts
+
+            expected = {
+                "cells": 784, "features": 322465,
+                "feature_repaired": 55, "coverage_repaired": 3,
+                "feature_issues": 0, "coverage_issues": 0,
+                "geotype16_total": 4406, "geotype16_issues": 0,
+            }
+            for k,v in expected.items():
+                if counts[k] != v:
+                    failures.append({"check":k,"expected":v,"actual":counts[k]})
+
+            afn = wkb_fn(cur, "chart_area_geometry")
+            cur.execute(f"""
+                SELECT f.id,f.object_class,f.geometry_repaired,
+                       f.bbox_min_lon,f.bbox_min_lat,f.bbox_max_lon,f.bbox_max_lat,
+                       {afn}(g.geom) AS wkb
+                FROM chart_feature f
+                JOIN chart_area_geometry g ON g.feature_id=f.id
+                WHERE f.dataset_id=%s
+            """,(a.dataset_id,))
+            area_total=area_valid=bbox_bad=0
+            for r in cur:
+                area_total += 1
+                try:
+                    g = load_wkb(as_bytes(r["wkb"]))
+                    if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid:
+                        failures.append({"check":"area_invalid","feature_id":r["id"],"object_class":r["object_class"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)})
+                        continue
+                    area_valid += 1
+                    dbb=(float(r["bbox_min_lon"]),float(r["bbox_min_lat"]),float(r["bbox_max_lon"]),float(r["bbox_max_lat"]))
+                    gb=tuple(float(x) for x in g.bounds)
+                    if any(abs(x-y)>a.bbox_tolerance for x,y in zip(dbb,gb)):
+                        bbox_bad += 1
+                        failures.append({"check":"area_bbox","feature_id":r["id"],"db_bbox":dbb,"geom_bbox":gb})
+                except Exception as e:
+                    failures.append({"check":"area_wkb","feature_id":r["id"],"error":repr(e)})
+
+            cfn = wkb_fn(cur, "chart_coverage")
+            cur.execute(f"""
+                SELECT id,cell_id,source_record_index,geometry_repaired,{cfn}(geom) AS wkb
+                FROM chart_coverage WHERE dataset_id=%s
+            """,(a.dataset_id,))
+            cov_total=cov_valid=0
+            for r in cur:
+                cov_total += 1
+                try:
+                    g = load_wkb(as_bytes(r["wkb"]))
+                    if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid:
+                        failures.append({"check":"coverage_invalid","coverage_id":r["id"],"cell_id":r["cell_id"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)})
+                        continue
+                    cov_valid += 1
+                except Exception as e:
+                    failures.append({"check":"coverage_wkb","coverage_id":r["id"],"error":repr(e)})
+
+            report["area"]={"total":area_total,"valid":area_valid,"bbox_mismatch":bbox_bad}
+            report["coverage"]={"total":cov_total,"valid":cov_valid}
+            conn.rollback()
+    finally:
+        conn.close()
+
+    report["failure_count"]=len(failures)
+    report["overall"]="PASS" if not failures else "FAIL"
+
+    with (outdir/"失败明细_v1.csv").open("w",encoding="utf-8-sig",newline="") as f:
+        keys=["check","expected","actual","feature_id","coverage_id","cell_id","object_class","geom_type","is_valid","db_bbox","geom_bbox","error"]
+        w=csv.DictWriter(f,fieldnames=keys,extrasaction="ignore"); w.writeheader(); w.writerows(failures)
+
+    (outdir/"最终验收报告_v1.json").write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8")
+    summary = f"""CM93 normalization 最终全库 Geometry 验收 v1
+Dataset ID             : {a.dataset_id}
+Cells                  : {report['counts']['cells']}
+Features               : {report['counts']['features']}
+Feature repaired       : {report['counts']['feature_repaired']}
+Coverage repaired      : {report['counts']['coverage_repaired']}
+Feature issues         : {report['counts']['feature_issues']}
+Coverage issues        : {report['counts']['coverage_issues']}
+Geotype16 total        : {report['counts']['geotype16_total']}
+Geotype16 issues       : {report['counts']['geotype16_issues']}
+Area total             : {report['area']['total']}
+Area valid             : {report['area']['valid']}
+Area bbox mismatch     : {report['area']['bbox_mismatch']}
+Coverage total         : {report['coverage']['total']}
+Coverage valid         : {report['coverage']['valid']}
+Failure count          : {report['failure_count']}
+OVERALL: {report['overall']}
+"""
+    (outdir/"最终验收摘要_v1.txt").write_text(summary,encoding="utf-8")
+    print(summary)
+    print("报告目录:", outdir.resolve())
+
+if __name__=="__main__":
+    main()

tools/update_manifest_sha256.ps1

@@ -0,0 +1,31 @@
+# CM95 tracked-file SHA256 manifest generator
+# 在仓库根目录执行：
+# powershell -ExecutionPolicy Bypass -File .\tools\update_manifest_sha256.ps1
+
+$ErrorActionPreference = "Stop"
+
+$root = (Get-Location).Path
+$out = Join-Path $root "MANIFEST.sha256"
+
+$files = git ls-files
+if ($LASTEXITCODE -ne 0) {
+    throw "git ls-files failed"
+}
+
+$lines = foreach ($rel in $files) {
+    if ($rel -eq "MANIFEST.sha256") {
+        continue
+    }
+
+    $path = Join-Path $root $rel
+    if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
+        continue
+    }
+
+    $hash = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
+    "$hash  $($rel -replace '\\','/')"
+}
+
+$lines | Set-Content -LiteralPath $out -Encoding UTF8
+Write-Host "Updated: $out"
+Write-Host "Files: $($lines.Count)"