183 lines
8.4 KiB
Python
183 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
from __future__ import annotations
|
||
import argparse, csv, getpass, json, sys
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
try:
|
||
import pymysql
|
||
from pymysql.cursors import DictCursor
|
||
except ImportError:
|
||
print("缺少 pymysql：pip install pymysql")
|
||
sys.exit(2)
|
||
|
||
try:
|
||
try:
|
||
from shapely import from_wkb
|
||
def load_wkb(x): return from_wkb(x)
|
||
except ImportError:
|
||
from shapely import wkb
|
||
def load_wkb(x): return wkb.loads(x)
|
||
except ImportError:
|
||
print("缺少 shapely：pip install shapely")
|
||
sys.exit(2)
|
||
|
||
def scalar(cur, sql, params=()):
|
||
cur.execute(sql, params)
|
||
r = cur.fetchone()
|
||
return None if not r else next(iter(r.values()))
|
||
|
||
def as_bytes(v):
|
||
if isinstance(v, memoryview): return v.tobytes()
|
||
if isinstance(v, bytearray): return bytes(v)
|
||
return v
|
||
|
||
def wkb_fn(cur, table):
|
||
for fn in ("ST_AsBinary","ST_AsWKB","AsBinary","AsWKB"):
|
||
try:
|
||
cur.execute(f"SELECT {fn}(geom) AS g FROM {table} LIMIT 1")
|
||
cur.fetchone()
|
||
return fn
|
||
except Exception:
|
||
pass
|
||
raise RuntimeError(f"{table}: 无可用 Geometry->WKB 函数")
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser(description="CM93 normalization 最终全库 Geometry 验收")
|
||
ap.add_argument("--host", default="127.0.0.1")
|
||
ap.add_argument("--port", type=int, default=3306)
|
||
ap.add_argument("--user", default="root")
|
||
ap.add_argument("--password")
|
||
ap.add_argument("--database", default="cm93_chart")
|
||
ap.add_argument("--dataset-id", type=int, default=1)
|
||
ap.add_argument("--bbox-tolerance", type=float, default=5e-6)
|
||
a = ap.parse_args()
|
||
if a.password is None:
|
||
a.password = getpass.getpass("MySQL password: ")
|
||
|
||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
outdir = Path(f"cm93_final_geometry_audit_{stamp}")
|
||
outdir.mkdir(exist_ok=True)
|
||
|
||
conn = pymysql.connect(
|
||
host=a.host, port=a.port, user=a.user, password=a.password,
|
||
database=a.database, charset="utf8mb4",
|
||
cursorclass=DictCursor, autocommit=False,
|
||
connect_timeout=20, read_timeout=900
|
||
)
|
||
|
||
failures = []
|
||
report = {"dataset_id": a.dataset_id, "generated_at": datetime.now().isoformat()}
|
||
|
||
try:
|
||
with conn.cursor() as cur:
|
||
try: cur.execute("SET SESSION TRANSACTION READ ONLY")
|
||
except Exception: pass
|
||
try: cur.execute("START TRANSACTION READ ONLY")
|
||
except Exception: cur.execute("START TRANSACTION")
|
||
|
||
counts = {
|
||
"cells": int(scalar(cur,"SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s",(a.dataset_id,)) or 0),
|
||
"features": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s",(a.dataset_id,)) or 0),
|
||
"feature_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0),
|
||
"coverage_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0),
|
||
"feature_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0),
|
||
"coverage_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0),
|
||
"geotype16_total": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16",(a.dataset_id,)) or 0),
|
||
"geotype16_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16 AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0),
|
||
}
|
||
report["counts"] = counts
|
||
|
||
expected = {
|
||
"cells": 784, "features": 322465,
|
||
"feature_repaired": 55, "coverage_repaired": 3,
|
||
"feature_issues": 0, "coverage_issues": 0,
|
||
"geotype16_total": 4406, "geotype16_issues": 0,
|
||
}
|
||
for k,v in expected.items():
|
||
if counts[k] != v:
|
||
failures.append({"check":k,"expected":v,"actual":counts[k]})
|
||
|
||
afn = wkb_fn(cur, "chart_area_geometry")
|
||
cur.execute(f"""
|
||
SELECT f.id,f.object_class,f.geometry_repaired,
|
||
f.bbox_min_lon,f.bbox_min_lat,f.bbox_max_lon,f.bbox_max_lat,
|
||
{afn}(g.geom) AS wkb
|
||
FROM chart_feature f
|
||
JOIN chart_area_geometry g ON g.feature_id=f.id
|
||
WHERE f.dataset_id=%s
|
||
""",(a.dataset_id,))
|
||
area_total=area_valid=bbox_bad=0
|
||
for r in cur:
|
||
area_total += 1
|
||
try:
|
||
g = load_wkb(as_bytes(r["wkb"]))
|
||
if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid:
|
||
failures.append({"check":"area_invalid","feature_id":r["id"],"object_class":r["object_class"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)})
|
||
continue
|
||
area_valid += 1
|
||
dbb=(float(r["bbox_min_lon"]),float(r["bbox_min_lat"]),float(r["bbox_max_lon"]),float(r["bbox_max_lat"]))
|
||
gb=tuple(float(x) for x in g.bounds)
|
||
if any(abs(x-y)>a.bbox_tolerance for x,y in zip(dbb,gb)):
|
||
bbox_bad += 1
|
||
failures.append({"check":"area_bbox","feature_id":r["id"],"db_bbox":dbb,"geom_bbox":gb})
|
||
except Exception as e:
|
||
failures.append({"check":"area_wkb","feature_id":r["id"],"error":repr(e)})
|
||
|
||
cfn = wkb_fn(cur, "chart_coverage")
|
||
cur.execute(f"""
|
||
SELECT id,cell_id,source_record_index,geometry_repaired,{cfn}(geom) AS wkb
|
||
FROM chart_coverage WHERE dataset_id=%s
|
||
""",(a.dataset_id,))
|
||
cov_total=cov_valid=0
|
||
for r in cur:
|
||
cov_total += 1
|
||
try:
|
||
g = load_wkb(as_bytes(r["wkb"]))
|
||
if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid:
|
||
failures.append({"check":"coverage_invalid","coverage_id":r["id"],"cell_id":r["cell_id"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)})
|
||
continue
|
||
cov_valid += 1
|
||
except Exception as e:
|
||
failures.append({"check":"coverage_wkb","coverage_id":r["id"],"error":repr(e)})
|
||
|
||
report["area"]={"total":area_total,"valid":area_valid,"bbox_mismatch":bbox_bad}
|
||
report["coverage"]={"total":cov_total,"valid":cov_valid}
|
||
conn.rollback()
|
||
finally:
|
||
conn.close()
|
||
|
||
report["failure_count"]=len(failures)
|
||
report["overall"]="PASS" if not failures else "FAIL"
|
||
|
||
with (outdir/"失败明细_v1.csv").open("w",encoding="utf-8-sig",newline="") as f:
|
||
keys=["check","expected","actual","feature_id","coverage_id","cell_id","object_class","geom_type","is_valid","db_bbox","geom_bbox","error"]
|
||
w=csv.DictWriter(f,fieldnames=keys,extrasaction="ignore"); w.writeheader(); w.writerows(failures)
|
||
|
||
(outdir/"最终验收报告_v1.json").write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8")
|
||
summary = f"""CM93 normalization 最终全库 Geometry 验收 v1
|
||
Dataset ID : {a.dataset_id}
|
||
Cells : {report['counts']['cells']}
|
||
Features : {report['counts']['features']}
|
||
Feature repaired : {report['counts']['feature_repaired']}
|
||
Coverage repaired : {report['counts']['coverage_repaired']}
|
||
Feature issues : {report['counts']['feature_issues']}
|
||
Coverage issues : {report['counts']['coverage_issues']}
|
||
Geotype16 total : {report['counts']['geotype16_total']}
|
||
Geotype16 issues : {report['counts']['geotype16_issues']}
|
||
Area total : {report['area']['total']}
|
||
Area valid : {report['area']['valid']}
|
||
Area bbox mismatch : {report['area']['bbox_mismatch']}
|
||
Coverage total : {report['coverage']['total']}
|
||
Coverage valid : {report['coverage']['valid']}
|
||
Failure count : {report['failure_count']}
|
||
OVERALL: {report['overall']}
|
||
"""
|
||
(outdir/"最终验收摘要_v1.txt").write_text(summary,encoding="utf-8")
|
||
print(summary)
|
||
print("报告目录:", outdir.resolve())
|
||
|
||
if __name__=="__main__":
|
||
main()
|