from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path from collections import defaultdict import math from typing import Any from shapely import from_wkb from .cm93_binary import parse_cell from .config import DbConfig from .database import ( connect, assert_mysql_compatible, server_version, mysql_compat_mode_for_connection, as_wkb_sql, ) from .dictionaries import load_object_dictionary, load_attribute_dictionary from .geometry import build_coverages, build_feature from .scanner import scan_cm93, CellFile @dataclass(frozen=True) class SourceCellStat: rel: str sha256: str file_size: int scale: str features: int attributes: int points: int lines: int areas: int coverages: int sounding_features: int sounding_points: int unknown_objects: int unknown_attributes: int @dataclass class PrecisionSample: rel: str record_index: int geometry_type: str expected_wkb: bytes point_index: int | None = None @dataclass class AuditResult: source_cells: int = 0 source_parse_failed: int = 0 db_missing_cells: int = 0 db_not_done_cells: int = 0 count_mismatches: int = 0 hash_mismatches: int = 0 unknown_objects: int = 0 unknown_attributes: int = 0 precision_checked: int = 0 precision_failed: int = 0 max_precision_delta_deg: float = 0.0 raw_preservation_mismatches: int = 0 geometry_validity_issues: int = 0 messages: list[str] = field(default_factory=list) @property def passed(self) -> bool: return not any((self.source_parse_failed,self.db_missing_cells,self.db_not_done_cells,self.count_mismatches,self.hash_mismatches,self.raw_preservation_mismatches,self.unknown_objects,self.unknown_attributes,self.precision_failed)) def as_dict(self) -> dict[str, Any]: return { "passed": self.passed, "source_cells": self.source_cells, "source_parse_failed": self.source_parse_failed, "db_missing_cells": self.db_missing_cells, "db_not_done_cells": self.db_not_done_cells, "count_mismatches": self.count_mismatches, "hash_mismatches": self.hash_mismatches, "raw_preservation_mismatches": self.raw_preservation_mismatches, "geometry_validity_issues": self.geometry_validity_issues, "unknown_objects": self.unknown_objects, "unknown_attributes": self.unknown_attributes, "precision_checked": self.precision_checked, "precision_failed": self.precision_failed, "max_precision_delta_deg": self.max_precision_delta_deg, "messages": self.messages, } def _rel(cf: CellFile) -> str: return str(cf.relative_path).replace("/", "\\") def _select_cells(scan, limit: int | None, scales: set[str] | None, cell_name: str | None): cells=list(scan.cells) if scales: cells=[c for c in cells if c.scale_code in scales] if cell_name: t=cell_name.lower() cells=[c for c in cells if c.path.name.lower()==t or c.path.stem.lower()==t] if limit is not None: cells=cells[:limit] return cells def _source_stat(cell, cf: CellFile) -> SourceCellStat: p=l=a=cov=sf=sp=0 for f in cell.features: if f.object_class == "_m_sor": cov += 1 continue if f.geometry_primitive == 1: p += 1 elif f.geometry_primitive == 2: l += 1 elif f.geometry_primitive == 4: a += 1 elif f.geometry_primitive == 8: sf += 1 if f.point3d_index is not None: sp += len(cell.point3d[f.point3d_index].points) return SourceCellStat( rel=_rel(cf),sha256=cell.sha256,file_size=cell.file_size,scale=cf.scale_code,features=len(cell.features),attributes=cell.attribute_count, points=p,lines=l,areas=a,coverages=cov,sounding_features=sf,sounding_points=sp, unknown_objects=len(cell.unknown_object_ids),unknown_attributes=len(cell.unknown_attribute_ids), ) def collect_source(root: Path, limit: int | None = None, scales: set[str] | None = None, cell_name: str | None = None, precision_samples: int = 0, validate_geometry: bool = False): scan=scan_cm93(root) if not scan.object_dictionary or not scan.attribute_dictionary: raise RuntimeError("CM93 字典未找到") objs={x.object_id:x for x in load_object_dictionary(scan.object_dictionary)} attrs={x.attribute_id:x for x in load_attribute_dictionary(scan.attribute_dictionary)} cells=_select_cells(scan,limit,scales,cell_name) stats: dict[str,SourceCellStat]={} failures: list[str]=[] samples: list[PrecisionSample]=[] geom_issues=0 quota=max(1, math.ceil(precision_samples / max(1,len({c.scale_code for c in cells})))) if precision_samples else 0 per_scale=defaultdict(int) for n,cf in enumerate(cells,1): try: cell=parse_cell(cf.path,objs,attrs) st=_source_stat(cell,cf); stats[st.rel]=st want = precision_samples and per_scale[cf.scale_code] < quota and len(samples) < precision_samples if want or validate_geometry: covs=build_coverages(cell) for f in cell.features: b=build_feature(cell,f,covs,validate=validate_geometry) if validate_geometry and b.geometry_issue: geom_issues += 1 if want and per_scale[cf.scale_code] < quota and len(samples) < precision_samples: if b.geom is not None and b.geometry_type in {"POINT","LINE","AREA"}: samples.append(PrecisionSample(st.rel,f.index,b.geometry_type,b.geom.wkb)) per_scale[cf.scale_code]+=1 elif b.geometry_type=="SOUNDING" and b.soundings: samples.append(PrecisionSample(st.rel,f.index,"SOUNDING",b.soundings[0][4].wkb,0)) per_scale[cf.scale_code]+=1 if n % 250 == 0 or n == len(cells): print(f"source audit [{n}/{len(cells)}]") except Exception as e: failures.append(f"{_rel(cf)}: {e}") return stats,failures,samples,geom_issues def _group_counts(conn, sql: str) -> dict[int,int]: with conn.cursor() as cur: cur.execute(sql) return {int(r["cell_id"]):int(r["n"]) for r in cur.fetchall()} def _dataset_id(conn, root: Path) -> int | None: with conn.cursor() as cur: cur.execute("SELECT id FROM chart_dataset WHERE source_type='CM93' AND source_root=%s ORDER BY id LIMIT 1", (str(root.resolve()),)) r=cur.fetchone(); return int(r["id"]) if r else None def _precision_db_geom(conn, cell_id: int, sample: PrecisionSample): db_mode = mysql_compat_mode_for_connection(conn) if sample.geometry_type == "SOUNDING": sql=(f"SELECT {as_wkb_sql(db_mode, 's.geom')} AS wkb FROM chart_feature f JOIN chart_sounding_point s ON s.feature_id=f.id " "WHERE f.cell_id=%s AND f.source_record_index=%s AND s.point_index=%s") params=(cell_id,sample.record_index,sample.point_index or 0) else: table={"POINT":"chart_point_geometry","LINE":"chart_line_geometry","AREA":"chart_area_geometry"}[sample.geometry_type] sql=(f"SELECT {as_wkb_sql(db_mode, 'g.geom')} AS wkb FROM chart_feature f JOIN {table} g ON g.feature_id=f.id " "WHERE f.cell_id=%s AND f.source_record_index=%s") params=(cell_id,sample.record_index) with conn.cursor() as cur: cur.execute(sql,params); r=cur.fetchone() return bytes(r["wkb"]) if r and r["wkb"] is not None else None def run_audit(cfg: DbConfig, root: Path, limit: int | None = None, scales: set[str] | None = None, cell_name: str | None = None, precision_samples: int = 128, tolerance_deg: float = 1e-12, validate_geometry: bool = False, source_only: bool = False, require_raw: bool = True) -> AuditResult: result=AuditResult() stats,failures,samples,geom_issues=collect_source(root,limit,scales,cell_name,precision_samples if not source_only else 0,validate_geometry) result.source_cells=len(stats)+len(failures) result.source_parse_failed=len(failures) result.messages.extend(f"SOURCE_PARSE_FAILED {m}" for m in failures[:100]) result.unknown_objects=sum(s.unknown_objects for s in stats.values()) result.unknown_attributes=sum(s.unknown_attributes for s in stats.values()) print(f"Source cells parsed: {len(stats)}, failed: {len(failures)}") print(f"Source features: {sum(s.features for s in stats.values())}") print(f"Source attributes: {sum(s.attributes for s in stats.values())}") result.geometry_validity_issues=geom_issues if validate_geometry: print(f"Source geometry issues (Shapely validity/structure): {geom_issues}") if source_only: return result conn=connect(cfg) try: assert_mysql_compatible(server_version(conn)) did=_dataset_id(conn,root) if did is None: result.db_missing_cells=len(stats) result.messages.append("数据库中找不到该 source_root 对应 dataset") return result with conn.cursor() as cur: cur.execute("SELECT id,source_relative_path,import_status,source_sha256,feature_count,attribute_count FROM chart_cell WHERE dataset_id=%s", (did,)) rows=cur.fetchall() db_by_rel={str(r["source_relative_path"]):r for r in rows} cellids=[int(r["id"]) for r in rows] fcnt=_group_counts(conn,f"SELECT cell_id,COUNT(*) n FROM chart_feature WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id") acnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_feature_attribute a JOIN chart_feature f ON f.id=a.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") pcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_point_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") lcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_line_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") arcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_area_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") ccnt=_group_counts(conn,f"SELECT cell_id,COUNT(*) n FROM chart_coverage WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id") scnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_sounding_point s JOIN chart_feature f ON f.id=s.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") rawbytes=_group_counts(conn,f"SELECT cell_id,SUM(OCTET_LENGTH(raw_blob)) n FROM chart_raw_record WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id") for rel,s in stats.items(): r=db_by_rel.get(rel) if not r: result.db_missing_cells+=1; result.messages.append(f"DB_MISSING {rel}"); continue cid=int(r["id"]) if r["import_status"]!="DONE": result.db_not_done_cells+=1; result.messages.append(f"DB_NOT_DONE {rel} status={r['import_status']}") if r.get("source_sha256") and str(r["source_sha256"]) != s.sha256: result.hash_mismatches+=1; result.messages.append(f"HASH_MISMATCH {rel}") actual=(fcnt.get(cid,0),acnt.get(cid,0),pcnt.get(cid,0),lcnt.get(cid,0),arcnt.get(cid,0),ccnt.get(cid,0),scnt.get(cid,0)) expected=(s.features,s.attributes,s.points,s.lines,s.areas,s.coverages,s.sounding_points) if actual != expected: result.count_mismatches+=1 result.messages.append(f"COUNT_MISMATCH {rel} expected={expected} actual={actual}") if require_raw and rawbytes.get(cid,0) != s.file_size: result.raw_preservation_mismatches += 1 result.messages.append(f"RAW_BYTES_MISMATCH {rel} expected={s.file_size} actual={rawbytes.get(cid,0)}") # Precision audit: source-derived geometry versus geometry round-tripped through MySQL. rel_to_id={k:int(v["id"]) for k,v in db_by_rel.items()} for sample in samples: cid=rel_to_id.get(sample.rel) if not cid: continue dbwkb=_precision_db_geom(conn,cid,sample) result.precision_checked+=1 if dbwkb is None: result.precision_failed+=1 result.messages.append(f"PRECISION_MISSING {sample.rel} record={sample.record_index}") continue expected=from_wkb(sample.expected_wkb); actual=from_wkb(dbwkb) try: ok=expected.equals_exact(actual,tolerance_deg) delta=expected.hausdorff_distance(actual) if not ok else 0.0 except Exception: ok=False; delta=float("inf") if math.isfinite(delta): result.max_precision_delta_deg=max(result.max_precision_delta_deg,float(delta)) if not ok: result.precision_failed+=1 result.messages.append(f"PRECISION_FAIL {sample.rel} record={sample.record_index} type={sample.geometry_type} delta_deg={delta}") finally: conn.close() return result def print_result(r: AuditResult) -> None: print("\n=== CM93 AUDIT ===") print(f"source_cells : {r.source_cells}") print(f"source_parse_failed : {r.source_parse_failed}") print(f"db_missing_cells : {r.db_missing_cells}") print(f"db_not_done_cells : {r.db_not_done_cells}") print(f"count_mismatches : {r.count_mismatches}") print(f"hash_mismatches : {r.hash_mismatches}") print(f"raw_preservation_bad : {r.raw_preservation_mismatches}") print(f"geometry_validity : {r.geometry_validity_issues} (informational)") print(f"unknown_objects : {r.unknown_objects}") print(f"unknown_attributes : {r.unknown_attributes}") print(f"precision_checked : {r.precision_checked}") print(f"precision_failed : {r.precision_failed}") print(f"max_precision_delta° : {r.max_precision_delta_deg:.16g}") print(f"RESULT : {'PASS' if r.passed else 'FAIL'}") if r.messages: print("\nDetails (max 100 shown):") for m in r.messages[:100]: print(" -",m)