#!/usr/bin/env python3 # -*- coding: utf-8 -*- from __future__ import annotations import argparse, csv, getpass, json, sys from datetime import datetime from pathlib import Path try: import pymysql from pymysql.cursors import DictCursor except ImportError: print("缺少 pymysql：pip install pymysql") sys.exit(2) try: try: from shapely import from_wkb def load_wkb(x): return from_wkb(x) except ImportError: from shapely import wkb def load_wkb(x): return wkb.loads(x) except ImportError: print("缺少 shapely：pip install shapely") sys.exit(2) def scalar(cur, sql, params=()): cur.execute(sql, params) r = cur.fetchone() return None if not r else next(iter(r.values())) def as_bytes(v): if isinstance(v, memoryview): return v.tobytes() if isinstance(v, bytearray): return bytes(v) return v def wkb_fn(cur, table): for fn in ("ST_AsBinary","ST_AsWKB","AsBinary","AsWKB"): try: cur.execute(f"SELECT {fn}(geom) AS g FROM {table} LIMIT 1") cur.fetchone() return fn except Exception: pass raise RuntimeError(f"{table}: 无可用 Geometry->WKB 函数") def main(): ap = argparse.ArgumentParser(description="CM93 normalization 最终全库 Geometry 验收") ap.add_argument("--host", default="127.0.0.1") ap.add_argument("--port", type=int, default=3306) ap.add_argument("--user", default="root") ap.add_argument("--password") ap.add_argument("--database", default="cm93_chart") ap.add_argument("--dataset-id", type=int, default=1) ap.add_argument("--bbox-tolerance", type=float, default=5e-6) a = ap.parse_args() if a.password is None: a.password = getpass.getpass("MySQL password: ") stamp = datetime.now().strftime("%Y%m%d_%H%M%S") outdir = Path(f"cm93_final_geometry_audit_{stamp}") outdir.mkdir(exist_ok=True) conn = pymysql.connect( host=a.host, port=a.port, user=a.user, password=a.password, database=a.database, charset="utf8mb4", cursorclass=DictCursor, autocommit=False, connect_timeout=20, read_timeout=900 ) failures = [] report = {"dataset_id": a.dataset_id, "generated_at": datetime.now().isoformat()} try: with conn.cursor() as cur: try: cur.execute("SET SESSION TRANSACTION READ ONLY") except Exception: pass try: cur.execute("START TRANSACTION READ ONLY") except Exception: cur.execute("START TRANSACTION") counts = { "cells": int(scalar(cur,"SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s",(a.dataset_id,)) or 0), "features": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s",(a.dataset_id,)) or 0), "feature_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0), "coverage_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0), "feature_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0), "coverage_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0), "geotype16_total": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16",(a.dataset_id,)) or 0), "geotype16_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16 AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0), } report["counts"] = counts expected = { "cells": 784, "features": 322465, "feature_repaired": 55, "coverage_repaired": 3, "feature_issues": 0, "coverage_issues": 0, "geotype16_total": 4406, "geotype16_issues": 0, } for k,v in expected.items(): if counts[k] != v: failures.append({"check":k,"expected":v,"actual":counts[k]}) afn = wkb_fn(cur, "chart_area_geometry") cur.execute(f""" SELECT f.id,f.object_class,f.geometry_repaired, f.bbox_min_lon,f.bbox_min_lat,f.bbox_max_lon,f.bbox_max_lat, {afn}(g.geom) AS wkb FROM chart_feature f JOIN chart_area_geometry g ON g.feature_id=f.id WHERE f.dataset_id=%s """,(a.dataset_id,)) area_total=area_valid=bbox_bad=0 for r in cur: area_total += 1 try: g = load_wkb(as_bytes(r["wkb"])) if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid: failures.append({"check":"area_invalid","feature_id":r["id"],"object_class":r["object_class"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)}) continue area_valid += 1 dbb=(float(r["bbox_min_lon"]),float(r["bbox_min_lat"]),float(r["bbox_max_lon"]),float(r["bbox_max_lat"])) gb=tuple(float(x) for x in g.bounds) if any(abs(x-y)>a.bbox_tolerance for x,y in zip(dbb,gb)): bbox_bad += 1 failures.append({"check":"area_bbox","feature_id":r["id"],"db_bbox":dbb,"geom_bbox":gb}) except Exception as e: failures.append({"check":"area_wkb","feature_id":r["id"],"error":repr(e)}) cfn = wkb_fn(cur, "chart_coverage") cur.execute(f""" SELECT id,cell_id,source_record_index,geometry_repaired,{cfn}(geom) AS wkb FROM chart_coverage WHERE dataset_id=%s """,(a.dataset_id,)) cov_total=cov_valid=0 for r in cur: cov_total += 1 try: g = load_wkb(as_bytes(r["wkb"])) if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid: failures.append({"check":"coverage_invalid","coverage_id":r["id"],"cell_id":r["cell_id"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)}) continue cov_valid += 1 except Exception as e: failures.append({"check":"coverage_wkb","coverage_id":r["id"],"error":repr(e)}) report["area"]={"total":area_total,"valid":area_valid,"bbox_mismatch":bbox_bad} report["coverage"]={"total":cov_total,"valid":cov_valid} conn.rollback() finally: conn.close() report["failure_count"]=len(failures) report["overall"]="PASS" if not failures else "FAIL" with (outdir/"失败明细_v1.csv").open("w",encoding="utf-8-sig",newline="") as f: keys=["check","expected","actual","feature_id","coverage_id","cell_id","object_class","geom_type","is_valid","db_bbox","geom_bbox","error"] w=csv.DictWriter(f,fieldnames=keys,extrasaction="ignore"); w.writeheader(); w.writerows(failures) (outdir/"最终验收报告_v1.json").write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8") summary = f"""CM93 normalization 最终全库 Geometry 验收 v1 Dataset ID : {a.dataset_id} Cells : {report['counts']['cells']} Features : {report['counts']['features']} Feature repaired : {report['counts']['feature_repaired']} Coverage repaired : {report['counts']['coverage_repaired']} Feature issues : {report['counts']['feature_issues']} Coverage issues : {report['counts']['coverage_issues']} Geotype16 total : {report['counts']['geotype16_total']} Geotype16 issues : {report['counts']['geotype16_issues']} Area total : {report['area']['total']} Area valid : {report['area']['valid']} Area bbox mismatch : {report['area']['bbox_mismatch']} Coverage total : {report['coverage']['total']} Coverage valid : {report['coverage']['valid']} Failure count : {report['failure_count']} OVERALL: {report['overall']} """ (outdir/"最终验收摘要_v1.txt").write_text(summary,encoding="utf-8") print(summary) print("报告目录:", outdir.resolve()) if __name__=="__main__": main()