302 lines
14 KiB
Python
302 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from collections import defaultdict
|
|
import math
|
|
from typing import Any
|
|
|
|
from shapely import from_wkb
|
|
|
|
from .cm93_binary import parse_cell
|
|
from .config import DbConfig
|
|
from .database import (
|
|
connect,
|
|
assert_mysql_compatible,
|
|
server_version,
|
|
mysql_compat_mode_for_connection,
|
|
as_wkb_sql,
|
|
)
|
|
from .dictionaries import load_object_dictionary, load_attribute_dictionary
|
|
from .geometry import build_coverages, build_feature
|
|
from .scanner import scan_cm93, CellFile
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceCellStat:
|
|
rel: str
|
|
sha256: str
|
|
file_size: int
|
|
scale: str
|
|
features: int
|
|
attributes: int
|
|
points: int
|
|
lines: int
|
|
areas: int
|
|
coverages: int
|
|
sounding_features: int
|
|
sounding_points: int
|
|
unknown_objects: int
|
|
unknown_attributes: int
|
|
|
|
|
|
@dataclass
|
|
class PrecisionSample:
|
|
rel: str
|
|
record_index: int
|
|
geometry_type: str
|
|
expected_wkb: bytes
|
|
point_index: int | None = None
|
|
|
|
|
|
@dataclass
|
|
class AuditResult:
|
|
source_cells: int = 0
|
|
source_parse_failed: int = 0
|
|
db_missing_cells: int = 0
|
|
db_not_done_cells: int = 0
|
|
count_mismatches: int = 0
|
|
hash_mismatches: int = 0
|
|
unknown_objects: int = 0
|
|
unknown_attributes: int = 0
|
|
precision_checked: int = 0
|
|
precision_failed: int = 0
|
|
max_precision_delta_deg: float = 0.0
|
|
raw_preservation_mismatches: int = 0
|
|
geometry_validity_issues: int = 0
|
|
messages: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return not any((self.source_parse_failed,self.db_missing_cells,self.db_not_done_cells,self.count_mismatches,self.hash_mismatches,self.raw_preservation_mismatches,self.unknown_objects,self.unknown_attributes,self.precision_failed))
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"passed": self.passed,
|
|
"source_cells": self.source_cells,
|
|
"source_parse_failed": self.source_parse_failed,
|
|
"db_missing_cells": self.db_missing_cells,
|
|
"db_not_done_cells": self.db_not_done_cells,
|
|
"count_mismatches": self.count_mismatches,
|
|
"hash_mismatches": self.hash_mismatches,
|
|
"raw_preservation_mismatches": self.raw_preservation_mismatches,
|
|
"geometry_validity_issues": self.geometry_validity_issues,
|
|
"unknown_objects": self.unknown_objects,
|
|
"unknown_attributes": self.unknown_attributes,
|
|
"precision_checked": self.precision_checked,
|
|
"precision_failed": self.precision_failed,
|
|
"max_precision_delta_deg": self.max_precision_delta_deg,
|
|
"messages": self.messages,
|
|
}
|
|
|
|
|
|
def _rel(cf: CellFile) -> str:
|
|
return str(cf.relative_path).replace("/", "\\")
|
|
|
|
|
|
def _select_cells(scan, limit: int | None, scales: set[str] | None, cell_name: str | None):
|
|
cells=list(scan.cells)
|
|
if scales:
|
|
cells=[c for c in cells if c.scale_code in scales]
|
|
if cell_name:
|
|
t=cell_name.lower()
|
|
cells=[c for c in cells if c.path.name.lower()==t or c.path.stem.lower()==t]
|
|
if limit is not None:
|
|
cells=cells[:limit]
|
|
return cells
|
|
|
|
|
|
def _source_stat(cell, cf: CellFile) -> SourceCellStat:
|
|
p=l=a=cov=sf=sp=0
|
|
for f in cell.features:
|
|
if f.object_class == "_m_sor":
|
|
cov += 1
|
|
continue
|
|
if f.geometry_primitive == 1: p += 1
|
|
elif f.geometry_primitive == 2: l += 1
|
|
elif f.geometry_primitive == 4: a += 1
|
|
elif f.geometry_primitive == 8:
|
|
sf += 1
|
|
if f.point3d_index is not None:
|
|
sp += len(cell.point3d[f.point3d_index].points)
|
|
return SourceCellStat(
|
|
rel=_rel(cf),sha256=cell.sha256,file_size=cell.file_size,scale=cf.scale_code,features=len(cell.features),attributes=cell.attribute_count,
|
|
points=p,lines=l,areas=a,coverages=cov,sounding_features=sf,sounding_points=sp,
|
|
unknown_objects=len(cell.unknown_object_ids),unknown_attributes=len(cell.unknown_attribute_ids),
|
|
)
|
|
|
|
|
|
def collect_source(root: Path, limit: int | None = None, scales: set[str] | None = None, cell_name: str | None = None,
|
|
precision_samples: int = 0, validate_geometry: bool = False):
|
|
scan=scan_cm93(root)
|
|
if not scan.object_dictionary or not scan.attribute_dictionary:
|
|
raise RuntimeError("CM93 字典未找到")
|
|
objs={x.object_id:x for x in load_object_dictionary(scan.object_dictionary)}
|
|
attrs={x.attribute_id:x for x in load_attribute_dictionary(scan.attribute_dictionary)}
|
|
cells=_select_cells(scan,limit,scales,cell_name)
|
|
stats: dict[str,SourceCellStat]={}
|
|
failures: list[str]=[]
|
|
samples: list[PrecisionSample]=[]
|
|
geom_issues=0
|
|
quota=max(1, math.ceil(precision_samples / max(1,len({c.scale_code for c in cells})))) if precision_samples else 0
|
|
per_scale=defaultdict(int)
|
|
|
|
for n,cf in enumerate(cells,1):
|
|
try:
|
|
cell=parse_cell(cf.path,objs,attrs)
|
|
st=_source_stat(cell,cf); stats[st.rel]=st
|
|
want = precision_samples and per_scale[cf.scale_code] < quota and len(samples) < precision_samples
|
|
if want or validate_geometry:
|
|
covs=build_coverages(cell)
|
|
for f in cell.features:
|
|
b=build_feature(cell,f,covs,validate=validate_geometry)
|
|
if validate_geometry and b.geometry_issue:
|
|
geom_issues += 1
|
|
if want and per_scale[cf.scale_code] < quota and len(samples) < precision_samples:
|
|
if b.geom is not None and b.geometry_type in {"POINT","LINE","AREA"}:
|
|
samples.append(PrecisionSample(st.rel,f.index,b.geometry_type,b.geom.wkb))
|
|
per_scale[cf.scale_code]+=1
|
|
elif b.geometry_type=="SOUNDING" and b.soundings:
|
|
samples.append(PrecisionSample(st.rel,f.index,"SOUNDING",b.soundings[0][4].wkb,0))
|
|
per_scale[cf.scale_code]+=1
|
|
if n % 250 == 0 or n == len(cells):
|
|
print(f"source audit [{n}/{len(cells)}]")
|
|
except Exception as e:
|
|
failures.append(f"{_rel(cf)}: {e}")
|
|
return stats,failures,samples,geom_issues
|
|
|
|
|
|
def _group_counts(conn, sql: str) -> dict[int,int]:
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql)
|
|
return {int(r["cell_id"]):int(r["n"]) for r in cur.fetchall()}
|
|
|
|
|
|
def _dataset_id(conn, root: Path) -> int | None:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT id FROM chart_dataset WHERE source_type='CM93' AND source_root=%s ORDER BY id LIMIT 1", (str(root.resolve()),))
|
|
r=cur.fetchone(); return int(r["id"]) if r else None
|
|
|
|
|
|
def _precision_db_geom(conn, cell_id: int, sample: PrecisionSample):
|
|
db_mode = mysql_compat_mode_for_connection(conn)
|
|
if sample.geometry_type == "SOUNDING":
|
|
sql=(f"SELECT {as_wkb_sql(db_mode, 's.geom')} AS wkb FROM chart_feature f JOIN chart_sounding_point s ON s.feature_id=f.id "
|
|
"WHERE f.cell_id=%s AND f.source_record_index=%s AND s.point_index=%s")
|
|
params=(cell_id,sample.record_index,sample.point_index or 0)
|
|
else:
|
|
table={"POINT":"chart_point_geometry","LINE":"chart_line_geometry","AREA":"chart_area_geometry"}[sample.geometry_type]
|
|
sql=(f"SELECT {as_wkb_sql(db_mode, 'g.geom')} AS wkb FROM chart_feature f JOIN {table} g ON g.feature_id=f.id "
|
|
"WHERE f.cell_id=%s AND f.source_record_index=%s")
|
|
params=(cell_id,sample.record_index)
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql,params); r=cur.fetchone()
|
|
return bytes(r["wkb"]) if r and r["wkb"] is not None else None
|
|
|
|
|
|
def run_audit(cfg: DbConfig, root: Path, limit: int | None = None, scales: set[str] | None = None, cell_name: str | None = None,
|
|
precision_samples: int = 128, tolerance_deg: float = 1e-12, validate_geometry: bool = False, source_only: bool = False, require_raw: bool = True) -> AuditResult:
|
|
result=AuditResult()
|
|
stats,failures,samples,geom_issues=collect_source(root,limit,scales,cell_name,precision_samples if not source_only else 0,validate_geometry)
|
|
result.source_cells=len(stats)+len(failures)
|
|
result.source_parse_failed=len(failures)
|
|
result.messages.extend(f"SOURCE_PARSE_FAILED {m}" for m in failures[:100])
|
|
result.unknown_objects=sum(s.unknown_objects for s in stats.values())
|
|
result.unknown_attributes=sum(s.unknown_attributes for s in stats.values())
|
|
print(f"Source cells parsed: {len(stats)}, failed: {len(failures)}")
|
|
print(f"Source features: {sum(s.features for s in stats.values())}")
|
|
print(f"Source attributes: {sum(s.attributes for s in stats.values())}")
|
|
result.geometry_validity_issues=geom_issues
|
|
if validate_geometry:
|
|
print(f"Source geometry issues (Shapely validity/structure): {geom_issues}")
|
|
if source_only:
|
|
return result
|
|
|
|
conn=connect(cfg)
|
|
try:
|
|
assert_mysql_compatible(server_version(conn))
|
|
did=_dataset_id(conn,root)
|
|
if did is None:
|
|
result.db_missing_cells=len(stats)
|
|
result.messages.append("数据库中找不到该 source_root 对应 dataset")
|
|
return result
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT id,source_relative_path,import_status,source_sha256,feature_count,attribute_count FROM chart_cell WHERE dataset_id=%s", (did,))
|
|
rows=cur.fetchall()
|
|
db_by_rel={str(r["source_relative_path"]):r for r in rows}
|
|
cellids=[int(r["id"]) for r in rows]
|
|
fcnt=_group_counts(conn,f"SELECT cell_id,COUNT(*) n FROM chart_feature WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id")
|
|
acnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_feature_attribute a JOIN chart_feature f ON f.id=a.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id")
|
|
pcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_point_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id")
|
|
lcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_line_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id")
|
|
arcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_area_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id")
|
|
ccnt=_group_counts(conn,f"SELECT cell_id,COUNT(*) n FROM chart_coverage WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id")
|
|
scnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_sounding_point s JOIN chart_feature f ON f.id=s.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id")
|
|
rawbytes=_group_counts(conn,f"SELECT cell_id,SUM(OCTET_LENGTH(raw_blob)) n FROM chart_raw_record WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id")
|
|
|
|
for rel,s in stats.items():
|
|
r=db_by_rel.get(rel)
|
|
if not r:
|
|
result.db_missing_cells+=1; result.messages.append(f"DB_MISSING {rel}"); continue
|
|
cid=int(r["id"])
|
|
if r["import_status"]!="DONE":
|
|
result.db_not_done_cells+=1; result.messages.append(f"DB_NOT_DONE {rel} status={r['import_status']}")
|
|
if r.get("source_sha256") and str(r["source_sha256"]) != s.sha256:
|
|
result.hash_mismatches+=1; result.messages.append(f"HASH_MISMATCH {rel}")
|
|
actual=(fcnt.get(cid,0),acnt.get(cid,0),pcnt.get(cid,0),lcnt.get(cid,0),arcnt.get(cid,0),ccnt.get(cid,0),scnt.get(cid,0))
|
|
expected=(s.features,s.attributes,s.points,s.lines,s.areas,s.coverages,s.sounding_points)
|
|
if actual != expected:
|
|
result.count_mismatches+=1
|
|
result.messages.append(f"COUNT_MISMATCH {rel} expected={expected} actual={actual}")
|
|
if require_raw and rawbytes.get(cid,0) != s.file_size:
|
|
result.raw_preservation_mismatches += 1
|
|
result.messages.append(f"RAW_BYTES_MISMATCH {rel} expected={s.file_size} actual={rawbytes.get(cid,0)}")
|
|
|
|
# Precision audit: source-derived geometry versus geometry round-tripped through MySQL.
|
|
rel_to_id={k:int(v["id"]) for k,v in db_by_rel.items()}
|
|
for sample in samples:
|
|
cid=rel_to_id.get(sample.rel)
|
|
if not cid:
|
|
continue
|
|
dbwkb=_precision_db_geom(conn,cid,sample)
|
|
result.precision_checked+=1
|
|
if dbwkb is None:
|
|
result.precision_failed+=1
|
|
result.messages.append(f"PRECISION_MISSING {sample.rel} record={sample.record_index}")
|
|
continue
|
|
expected=from_wkb(sample.expected_wkb); actual=from_wkb(dbwkb)
|
|
try:
|
|
ok=expected.equals_exact(actual,tolerance_deg)
|
|
delta=expected.hausdorff_distance(actual) if not ok else 0.0
|
|
except Exception:
|
|
ok=False; delta=float("inf")
|
|
if math.isfinite(delta):
|
|
result.max_precision_delta_deg=max(result.max_precision_delta_deg,float(delta))
|
|
if not ok:
|
|
result.precision_failed+=1
|
|
result.messages.append(f"PRECISION_FAIL {sample.rel} record={sample.record_index} type={sample.geometry_type} delta_deg={delta}")
|
|
finally:
|
|
conn.close()
|
|
return result
|
|
|
|
|
|
def print_result(r: AuditResult) -> None:
|
|
print("\n=== CM93 AUDIT ===")
|
|
print(f"source_cells : {r.source_cells}")
|
|
print(f"source_parse_failed : {r.source_parse_failed}")
|
|
print(f"db_missing_cells : {r.db_missing_cells}")
|
|
print(f"db_not_done_cells : {r.db_not_done_cells}")
|
|
print(f"count_mismatches : {r.count_mismatches}")
|
|
print(f"hash_mismatches : {r.hash_mismatches}")
|
|
print(f"raw_preservation_bad : {r.raw_preservation_mismatches}")
|
|
print(f"geometry_validity : {r.geometry_validity_issues} (informational)")
|
|
print(f"unknown_objects : {r.unknown_objects}")
|
|
print(f"unknown_attributes : {r.unknown_attributes}")
|
|
print(f"precision_checked : {r.precision_checked}")
|
|
print(f"precision_failed : {r.precision_failed}")
|
|
print(f"max_precision_delta° : {r.max_precision_delta_deg:.16g}")
|
|
print(f"RESULT : {'PASS' if r.passed else 'FAIL'}")
|
|
if r.messages:
|
|
print("\nDetails (max 100 shown):")
|
|
for m in r.messages[:100]: print(" -",m)
|